[root@compiler00 hbase]# bin/hbase --config ~/hbase-conf/ org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList loop 1 1 2000000 /tmp/it_16224 1 -m unbalance 2016-08-27 16:32:27,770 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2016-08-27 16:32:28,222 INFO [main] hbase.HBaseClusterManager: Running with SSH user [root] and options [ -t] 2016-08-27 16:32:28,370 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x5f52f873 connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:32:28,380 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT 2016-08-27 16:32:28,380 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=compiler00.ncku.edu.tw 2016-08-27 16:32:28,380 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.7.0_79 2016-08-27 16:32:28,380 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation 2016-08-27 16:32:28,380 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/java/jdk1.7.0_79/jre 2016-08-27 16:32:28,380 INFO [main] zookeeper.ZooKeeper: Client environment:java.class.path=/root/hbase-conf/:/usr/java/latest//lib/tools.jar:/root/hbase/bin/../hbase-server/target:/root/hbase/hbase-it/target/hbase-it-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-common/target/hbase-common-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-protocol/target/hbase-protocol-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-client/target/hbase-client-1.4.0-SNAPSHOT.jar:/root/.m2/repository/org/jruby/jcodings/jcodings/1.0.8/jcodings-1.0.8.jar:/root/.m2/repository/org/jruby/joni/joni/2.1.2/joni-2.1.2.jar:/root/.m2/repository/com/google/guava/guava/12.0.1/guava-12.0.1.jar:/root/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/root/.m2/repository/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar:/root/.m2/repository/commons-logging/commons-logging/1.2/commons-logging-1.2.jar:/root/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/root/.m2/repository/org/apache/commons/commons-math/2.2/commons-math-2.2.jar:/root/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/root/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.5.1/hadoop-mapreduce-client-core-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-common/2.5.1/hadoop-yarn-common-2.5.1.jar:/root/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/root/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/root/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/root/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/root/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/root/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/root/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/root/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/root/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/root/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/root/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-annotations/2.5.1/hadoop-annotations-2.5.1.jar:/usr/java/jdk1.7.0_79/jre/../lib/tools.jar:/root/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/root/.m2/repository/org/apache/hadoop/hadoop-common/2.5.1/hadoop-common-2.5.1.jar:/root/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/root/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/root/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/root/.m2/repository/org/codehaus/jettison/jettison/1.3.3/jettison-1.3.3.jar:/root/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/root/.m2/repository/commons-el/commons-el/1.0/commons-el-1.0.jar:/root/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/root/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/root/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/root/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/root/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/root/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/root/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/root/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/root/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/root/hbase/hbase-server/target/hbase-server-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-procedure/target/hbase-procedure-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-prefix-tree/target/hbase-prefix-tree-1.4.0-SNAPSHOT.jar:/root/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/root/.m2/repository/commons-codec/commons-codec/1.9/commons-codec-1.9.jar:/root/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/root/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/root/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/root/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/root/.m2/repository/asm/asm/3.1/asm-3.1.jar:/root/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/root/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/root/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/root/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/root/.m2/repository/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/root/.m2/repository/org/mortbay/jetty/jsp-2.1/6.1.14/jsp-2.1-6.1.14.jar:/root/.m2/repository/org/mortbay/jetty/jsp-api-2.1/6.1.14/jsp-api-2.1-6.1.14.jar:/root/.m2/repository/org/mortbay/jetty/servlet-api-2.5/6.1.14/servlet-api-2.5-6.1.14.jar:/root/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/root/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/root/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/root/.m2/repository/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar:/root/.m2/repository/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar:/root/.m2/repository/org/jamon/jamon-runtime/2.4.1/jamon-runtime-2.4.1.jar:/root/.m2/repository/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/root/.m2/repository/com/lmax/disruptor/3.3.0/disruptor-3.3.0.jar:/root/.m2/repository/org/apache/hadoop/hadoop-auth/2.5.1/hadoop-auth-2.5.1.jar:/root/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/root/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/root/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/root/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/root/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/root/.m2/repository/org/apache/hadoop/hadoop-client/2.5.1/hadoop-client-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-app/2.5.1/hadoop-mapreduce-client-app-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-api/2.5.1/hadoop-yarn-api-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.5.1/hadoop-hdfs-2.5.1.jar:/root/.m2/repository/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/root/hbase/hbase-thrift/target/hbase-thrift-1.4.0-SNAPSHOT.jar:/root/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/root/.m2/repository/org/apache/thrift/libthrift/0.9.3/libthrift-0.9.3.jar:/root/.m2/repository/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar:/root/.m2/repository/org/slf4j/slf4j-api/1.7.7/slf4j-api-1.7.7.jar:/root/hbase/hbase-hadoop-compat/target/hbase-hadoop-compat-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-hadoop2-compat/target/hbase-hadoop2-compat-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-annotations/target/hbase-annotations-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-shell/target/hbase-shell-1.4.0-SNAPSHOT.jar:/root/.m2/repository/org/jruby/jruby-complete/1.6.8/jruby-complete-1.6.8.jar:/root/hbase/hbase-external-blockcache/target/hbase-external-blockcache-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-testing-util/target/hbase-testing-util-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-common/target/hbase-common-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-annotations/target/hbase-annotations-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-server/target/hbase-server-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-hadoop-compat/target/hbase-hadoop-compat-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-hadoop2-compat/target/hbase-hadoop2-compat-1.4.0-SNAPSHOT-tests.jar:/root/.m2/repository/org/slf4j/slf4j-log4j12/1.7.7/slf4j-log4j12-1.7.7.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.5.1/hadoop-mapreduce-client-jobclient-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/2.5.1/hadoop-mapreduce-client-common-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-client/2.5.1/hadoop-yarn-client-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-common/2.5.1/hadoop-yarn-server-common-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.5.1/hadoop-mapreduce-client-shuffle-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.5.1/hadoop-yarn-server-nodemanager-2.5.1.jar:/root/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/root/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.5.1/hadoop-hdfs-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-minicluster/2.5.1/hadoop-minicluster-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-common/2.5.1/hadoop-common-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-tests/2.5.1/hadoop-yarn-server-tests-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.5.1/hadoop-yarn-server-resourcemanager-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.5.1/hadoop-yarn-server-applicationhistoryservice-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.5.1/hadoop-yarn-server-web-proxy-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.5.1/hadoop-mapreduce-client-jobclient-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-hs/2.5.1/hadoop-mapreduce-client-hs-2.5.1.jar:/root/hbase/hbase-resource-bundle/target/hbase-resource-bundle-1.4.0-SNAPSHOT.jar:/root/.m2/repository/com/github/stephenc/findbugs/findbugs-annotations/1.3.9-1/findbugs-annotations-1.3.9-1.jar:/root/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/root/.m2/repository/junit/junit/4.12/junit-4.12.jar:/root/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar:/root/.m2/repository/org/mockito/mockito-all/1.10.8/mockito-all-1.10.8.jar:/root/hbase/bin/../lib/*.jar: 2016-08-27 16:32:28,381 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=3.10.0-327.el7.x86_64 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=root 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/root 2016-08-27 16:32:28,382 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/root/hbase 2016-08-27 16:32:28,384 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@4636f0e0 2016-08-27 16:32:28,421 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase00.ncku.edu.tw/192.168.103.142:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:32:28,430 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase00.ncku.edu.tw/192.168.103.142:2181, initiating session 2016-08-27 16:32:28,444 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase00.ncku.edu.tw/192.168.103.142:2181, sessionid = 0x56cbb364240045, negotiated timeout = 90000 2016-08-27 16:32:29,867 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x3cc5d56a connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:32:29,867 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@603e4151 2016-08-27 16:32:29,868 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase00.ncku.edu.tw/192.168.103.142:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:32:29,869 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase00.ncku.edu.tw/192.168.103.142:2181, initiating session 2016-08-27 16:32:29,874 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase00.ncku.edu.tw/192.168.103.142:2181, sessionid = 0x56cbb364240046, negotiated timeout = 90000 2016-08-27 16:32:29,918 INFO [main] policies.Policy: Using ChaosMonkey Policy: class org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy, period: 65000 2016-08-27 16:32:29,921 INFO [main] test.IntegrationTestBigLinkedList$Loop: Running Loop with args:[1, 1, 2000000, /tmp/it_16224, 1] 2016-08-27 16:32:29,921 INFO [main] test.IntegrationTestBigLinkedList$Loop: Starting iteration = 0 2016-08-27 16:32:29,921 INFO [Thread-4] policies.Policy: Sleeping for 61343 to add jitter 2016-08-27 16:32:29,931 INFO [main] test.IntegrationTestBigLinkedList$Generator: Running RandomInputGenerator with numMappers=1, numNodes=2000000 2016-08-27 16:32:30,051 INFO [main] client.RMProxy: Connecting to ResourceManager at hbase00/192.168.103.142:8032 2016-08-27 16:32:31,122 INFO [main] mapreduce.JobSubmitter: number of splits:1 2016-08-27 16:32:31,438 INFO [main] mapreduce.JobSubmitter: Submitting tokens for job: job_1472292515371_0044 2016-08-27 16:32:31,682 INFO [main] impl.YarnClientImpl: Submitted application application_1472292515371_0044 2016-08-27 16:32:31,750 INFO [main] mapreduce.Job: The url to track the job: http://hbase00:8088/proxy/application_1472292515371_0044/ 2016-08-27 16:32:31,752 INFO [main] mapreduce.Job: Running job: job_1472292515371_0044 2016-08-27 16:32:38,005 INFO [main] mapreduce.Job: Job job_1472292515371_0044 running in uber mode : false 2016-08-27 16:32:38,008 INFO [main] mapreduce.Job: map 0% reduce 0% 2016-08-27 16:32:49,178 INFO [main] mapreduce.Job: map 70% reduce 0% 2016-08-27 16:32:51,232 INFO [main] mapreduce.Job: map 100% reduce 0% 2016-08-27 16:32:52,262 INFO [main] mapreduce.Job: Job job_1472292515371_0044 completed successfully 2016-08-27 16:32:52,433 INFO [main] mapreduce.Job: Counters: 30 File System Counters FILE: Number of bytes read=0 FILE: Number of bytes written=122226 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=109 HDFS: Number of bytes written=56555635 HDFS: Number of read operations=4 HDFS: Number of large read operations=0 HDFS: Number of write operations=2 Job Counters Launched map tasks=1 Other local map tasks=1 Total time spent by all maps in occupied slots (ms)=22480 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=11240 Total vcore-seconds taken by all map tasks=22480 Total megabyte-seconds taken by all map tasks=23019520 Map-Reduce Framework Map input records=2000000 Map output records=2000000 Input split bytes=109 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=116 CPU time spent (ms)=11020 Physical memory (bytes) snapshot=334278656 Virtual memory (bytes) snapshot=4175433728 Total committed heap usage (bytes)=438829056 File Input Format Counters Bytes Read=0 File Output Format Counters Bytes Written=56555635 2016-08-27 16:32:52,437 INFO [main] test.IntegrationTestBigLinkedList$Generator: Running Generator with numMappers=1, numNodes=2000000 2016-08-27 16:32:52,439 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x16ac0be1 connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:32:52,439 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@793fec67 2016-08-27 16:32:52,440 INFO [main-SendThread(hbase02.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase02.ncku.edu.tw/192.168.103.144:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:32:52,444 INFO [main-SendThread(hbase02.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase02.ncku.edu.tw/192.168.103.144:2181, initiating session 2016-08-27 16:32:52,450 INFO [main-SendThread(hbase02.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase02.ncku.edu.tw/192.168.103.144:2181, sessionid = 0x256cbb3643f004d, negotiated timeout = 90000 2016-08-27 16:32:52,719 INFO [main] test.IntegrationTestBigLinkedList$Generator: Number of live regionservers: 6, pre-splitting table into 18 regions (default regions per server: 3) 2016-08-27 16:32:55,041 INFO [main] client.HBaseAdmin: Created IntegrationTestBigLinkedList 2016-08-27 16:32:55,041 INFO [main] client.ConnectionManager$HConnectionImplementation: Closing master protocol: MasterService 2016-08-27 16:32:55,044 INFO [main] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x256cbb3643f004d 2016-08-27 16:32:55,064 INFO [main] zookeeper.ZooKeeper: Session: 0x256cbb3643f004d closed 2016-08-27 16:32:55,065 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down 2016-08-27 16:32:55,168 INFO [main] client.RMProxy: Connecting to ResourceManager at hbase00/192.168.103.142:8032 2016-08-27 16:32:56,047 INFO [main] input.FileInputFormat: Total input paths to process : 1 2016-08-27 16:32:56,124 INFO [main] mapreduce.JobSubmitter: number of splits:1 2016-08-27 16:32:56,190 INFO [main] mapreduce.JobSubmitter: Submitting tokens for job: job_1472292515371_0045 2016-08-27 16:32:56,289 INFO [main] impl.YarnClientImpl: Submitted application application_1472292515371_0045 2016-08-27 16:32:56,295 INFO [main] mapreduce.Job: The url to track the job: http://hbase00:8088/proxy/application_1472292515371_0045/ 2016-08-27 16:32:56,295 INFO [main] mapreduce.Job: Running job: job_1472292515371_0045 2016-08-27 16:33:04,800 INFO [main] mapreduce.Job: Job job_1472292515371_0045 running in uber mode : false 2016-08-27 16:33:04,801 INFO [main] mapreduce.Job: map 0% reduce 0% 2016-08-27 16:33:14,936 INFO [main] mapreduce.Job: map 50% reduce 0% 2016-08-27 16:33:31,302 INFO [Thread-4] actions.Action: Moving 12 regions from 4 servers to 2 different servers 2016-08-27 16:33:33,786 INFO [Thread-4] actions.Action: Killing region server:hbase06.ncku.edu.tw,16020,1472296349470 2016-08-27 16:33:33,787 INFO [Thread-4] hbase.HBaseCluster: Aborting RS: hbase06.ncku.edu.tw,16020,1472296349470 2016-08-27 16:33:33,789 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase06.ncku.edu.tw 2016-08-27 16:33:33,791 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -t root@hbase06.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL"] 2016-08-27 16:33:34,207 WARN [Thread-4] hbase.HBaseClusterManager: Remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase06.ncku.edu.tw failed at attempt 0. Retrying until maxAttempts: 5. Exception: stderr: Pseudo-terminal will not be allocated because stdin is not a terminal. sudo: sorry, you must have a tty to run sudo Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). , stdout: 2016-08-27 16:33:35,208 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase06.ncku.edu.tw 2016-08-27 16:33:35,209 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -t root@hbase06.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL"] 2016-08-27 16:33:35,594 WARN [Thread-4] hbase.HBaseClusterManager: Remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase06.ncku.edu.tw failed at attempt 1. Retrying until maxAttempts: 5. Exception: stderr: Pseudo-terminal will not be allocated because stdin is not a terminal. sudo: sorry, you must have a tty to run sudo Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). , stdout: 2016-08-27 16:33:37,595 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase06.ncku.edu.tw 2016-08-27 16:33:37,596 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -t root@hbase06.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL"] ^C2016-08-27 16:33:37,713 WARN [Thread-4] hbase.HBaseClusterManager: Remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase06.ncku.edu.tw failed at attempt 2. Retrying until maxAttempts: 5. Exception: stderr: Pseudo-terminal will not be allocated because stdin is not a terminal. , stdout: [root@compiler00 hbase]# ^C [root@compiler00 hbase]# ^C [root@compiler00 hbase]# ^C [root@compiler00 hbase]# ^C [root@compiler00 hbase]# ^C [root@compiler00 hbase]# bin/hbase --config ~/hbase-conf/ org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList loop 1 1 2000000 /tmp/it_16224 1 -m unbalance 2016-08-27 16:34:59,471 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2016-08-27 16:34:59,934 INFO [main] hbase.HBaseClusterManager: Running with SSH user [root] and options [ -tt] 2016-08-27 16:35:00,123 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x793a404a connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:35:00,138 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT 2016-08-27 16:35:00,138 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=compiler00.ncku.edu.tw 2016-08-27 16:35:00,138 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.7.0_79 2016-08-27 16:35:00,138 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation 2016-08-27 16:35:00,138 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/java/jdk1.7.0_79/jre 2016-08-27 16:35:00,138 INFO [main] zookeeper.ZooKeeper: Client environment:java.class.path=/root/hbase-conf/:/usr/java/latest//lib/tools.jar:/root/hbase/bin/../hbase-server/target:/root/hbase/hbase-it/target/hbase-it-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-common/target/hbase-common-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-protocol/target/hbase-protocol-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-client/target/hbase-client-1.4.0-SNAPSHOT.jar:/root/.m2/repository/org/jruby/jcodings/jcodings/1.0.8/jcodings-1.0.8.jar:/root/.m2/repository/org/jruby/joni/joni/2.1.2/joni-2.1.2.jar:/root/.m2/repository/com/google/guava/guava/12.0.1/guava-12.0.1.jar:/root/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/root/.m2/repository/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar:/root/.m2/repository/commons-logging/commons-logging/1.2/commons-logging-1.2.jar:/root/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/root/.m2/repository/org/apache/commons/commons-math/2.2/commons-math-2.2.jar:/root/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/root/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.5.1/hadoop-mapreduce-client-core-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-common/2.5.1/hadoop-yarn-common-2.5.1.jar:/root/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/root/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/root/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/root/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/root/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/root/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/root/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/root/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/root/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/root/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/root/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-annotations/2.5.1/hadoop-annotations-2.5.1.jar:/usr/java/jdk1.7.0_79/jre/../lib/tools.jar:/root/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/root/.m2/repository/org/apache/hadoop/hadoop-common/2.5.1/hadoop-common-2.5.1.jar:/root/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/root/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/root/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/root/.m2/repository/org/codehaus/jettison/jettison/1.3.3/jettison-1.3.3.jar:/root/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/root/.m2/repository/commons-el/commons-el/1.0/commons-el-1.0.jar:/root/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/root/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/root/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/root/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/root/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/root/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/root/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/root/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/root/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/root/hbase/hbase-server/target/hbase-server-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-procedure/target/hbase-procedure-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-prefix-tree/target/hbase-prefix-tree-1.4.0-SNAPSHOT.jar:/root/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/root/.m2/repository/commons-codec/commons-codec/1.9/commons-codec-1.9.jar:/root/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/root/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/root/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/root/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/root/.m2/repository/asm/asm/3.1/asm-3.1.jar:/root/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/root/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/root/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/root/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/root/.m2/repository/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/root/.m2/repository/org/mortbay/jetty/jsp-2.1/6.1.14/jsp-2.1-6.1.14.jar:/root/.m2/repository/org/mortbay/jetty/jsp-api-2.1/6.1.14/jsp-api-2.1-6.1.14.jar:/root/.m2/repository/org/mortbay/jetty/servlet-api-2.5/6.1.14/servlet-api-2.5-6.1.14.jar:/root/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/root/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/root/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/root/.m2/repository/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar:/root/.m2/repository/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar:/root/.m2/repository/org/jamon/jamon-runtime/2.4.1/jamon-runtime-2.4.1.jar:/root/.m2/repository/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/root/.m2/repository/com/lmax/disruptor/3.3.0/disruptor-3.3.0.jar:/root/.m2/repository/org/apache/hadoop/hadoop-auth/2.5.1/hadoop-auth-2.5.1.jar:/root/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/root/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/root/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/root/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/root/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/root/.m2/repository/org/apache/hadoop/hadoop-client/2.5.1/hadoop-client-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-app/2.5.1/hadoop-mapreduce-client-app-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-api/2.5.1/hadoop-yarn-api-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.5.1/hadoop-hdfs-2.5.1.jar:/root/.m2/repository/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/root/hbase/hbase-thrift/target/hbase-thrift-1.4.0-SNAPSHOT.jar:/root/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/root/.m2/repository/org/apache/thrift/libthrift/0.9.3/libthrift-0.9.3.jar:/root/.m2/repository/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar:/root/.m2/repository/org/slf4j/slf4j-api/1.7.7/slf4j-api-1.7.7.jar:/root/hbase/hbase-hadoop-compat/target/hbase-hadoop-compat-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-hadoop2-compat/target/hbase-hadoop2-compat-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-annotations/target/hbase-annotations-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-shell/target/hbase-shell-1.4.0-SNAPSHOT.jar:/root/.m2/repository/org/jruby/jruby-complete/1.6.8/jruby-complete-1.6.8.jar:/root/hbase/hbase-external-blockcache/target/hbase-external-blockcache-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-testing-util/target/hbase-testing-util-1.4.0-SNAPSHOT.jar:/root/hbase/hbase-common/target/hbase-common-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-annotations/target/hbase-annotations-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-server/target/hbase-server-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-hadoop-compat/target/hbase-hadoop-compat-1.4.0-SNAPSHOT-tests.jar:/root/hbase/hbase-hadoop2-compat/target/hbase-hadoop2-compat-1.4.0-SNAPSHOT-tests.jar:/root/.m2/repository/org/slf4j/slf4j-log4j12/1.7.7/slf4j-log4j12-1.7.7.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.5.1/hadoop-mapreduce-client-jobclient-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/2.5.1/hadoop-mapreduce-client-common-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-client/2.5.1/hadoop-yarn-client-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-common/2.5.1/hadoop-yarn-server-common-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.5.1/hadoop-mapreduce-client-shuffle-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.5.1/hadoop-yarn-server-nodemanager-2.5.1.jar:/root/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/root/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.5.1/hadoop-hdfs-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-minicluster/2.5.1/hadoop-minicluster-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-common/2.5.1/hadoop-common-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-tests/2.5.1/hadoop-yarn-server-tests-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.5.1/hadoop-yarn-server-resourcemanager-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.5.1/hadoop-yarn-server-applicationhistoryservice-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.5.1/hadoop-yarn-server-web-proxy-2.5.1.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.5.1/hadoop-mapreduce-client-jobclient-2.5.1-tests.jar:/root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-hs/2.5.1/hadoop-mapreduce-client-hs-2.5.1.jar:/root/hbase/hbase-resource-bundle/target/hbase-resource-bundle-1.4.0-SNAPSHOT.jar:/root/.m2/repository/com/github/stephenc/findbugs/findbugs-annotations/1.3.9-1/findbugs-annotations-1.3.9-1.jar:/root/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/root/.m2/repository/junit/junit/4.12/junit-4.12.jar:/root/.m2/repository/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar:/root/.m2/repository/org/mockito/mockito-all/1.10.8/mockito-all-1.10.8.jar:/root/hbase/bin/../lib/*.jar: 2016-08-27 16:35:00,139 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib 2016-08-27 16:35:00,139 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp 2016-08-27 16:35:00,139 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= 2016-08-27 16:35:00,140 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux 2016-08-27 16:35:00,140 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 2016-08-27 16:35:00,140 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=3.10.0-327.el7.x86_64 2016-08-27 16:35:00,140 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=root 2016-08-27 16:35:00,140 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/root 2016-08-27 16:35:00,140 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/root/hbase 2016-08-27 16:35:00,141 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@5ede1508 2016-08-27 16:35:00,182 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase00.ncku.edu.tw/192.168.103.142:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:35:00,191 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase00.ncku.edu.tw/192.168.103.142:2181, initiating session 2016-08-27 16:35:00,206 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase00.ncku.edu.tw/192.168.103.142:2181, sessionid = 0x56cbb364240048, negotiated timeout = 90000 2016-08-27 16:35:01,395 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x75cf7815 connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:35:01,396 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@7d4e720 2016-08-27 16:35:01,397 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase00.ncku.edu.tw/192.168.103.142:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:35:01,398 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase00.ncku.edu.tw/192.168.103.142:2181, initiating session 2016-08-27 16:35:01,402 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase00.ncku.edu.tw/192.168.103.142:2181, sessionid = 0x56cbb364240049, negotiated timeout = 90000 2016-08-27 16:35:01,443 INFO [main] policies.Policy: Using ChaosMonkey Policy: class org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy, period: 65000 2016-08-27 16:35:01,451 INFO [Thread-4] policies.Policy: Sleeping for 64431 to add jitter 2016-08-27 16:35:01,451 INFO [main] test.IntegrationTestBigLinkedList$Loop: Running Loop with args:[1, 1, 2000000, /tmp/it_16224, 1] 2016-08-27 16:35:01,452 INFO [main] test.IntegrationTestBigLinkedList$Loop: Starting iteration = 0 2016-08-27 16:35:01,464 INFO [main] test.IntegrationTestBigLinkedList$Generator: Running RandomInputGenerator with numMappers=1, numNodes=2000000 2016-08-27 16:35:01,577 INFO [main] client.RMProxy: Connecting to ResourceManager at hbase00/192.168.103.142:8032 2016-08-27 16:35:02,566 INFO [main] mapreduce.JobSubmitter: number of splits:1 2016-08-27 16:35:02,927 INFO [main] mapreduce.JobSubmitter: Submitting tokens for job: job_1472292515371_0046 2016-08-27 16:35:03,174 INFO [main] impl.YarnClientImpl: Submitted application application_1472292515371_0046 2016-08-27 16:35:03,237 INFO [main] mapreduce.Job: The url to track the job: http://hbase00:8088/proxy/application_1472292515371_0046/ 2016-08-27 16:35:03,238 INFO [main] mapreduce.Job: Running job: job_1472292515371_0046 2016-08-27 16:35:09,526 INFO [main] mapreduce.Job: Job job_1472292515371_0046 running in uber mode : false 2016-08-27 16:35:09,528 INFO [main] mapreduce.Job: map 0% reduce 0% 2016-08-27 16:35:20,680 INFO [main] mapreduce.Job: map 81% reduce 0% 2016-08-27 16:35:21,709 INFO [main] mapreduce.Job: map 100% reduce 0% 2016-08-27 16:35:22,735 INFO [main] mapreduce.Job: Job job_1472292515371_0046 completed successfully 2016-08-27 16:35:22,909 INFO [main] mapreduce.Job: Counters: 30 File System Counters FILE: Number of bytes read=0 FILE: Number of bytes written=122227 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=109 HDFS: Number of bytes written=56555635 HDFS: Number of read operations=4 HDFS: Number of large read operations=0 HDFS: Number of write operations=2 Job Counters Launched map tasks=1 Other local map tasks=1 Total time spent by all maps in occupied slots (ms)=20994 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=10497 Total vcore-seconds taken by all map tasks=20994 Total megabyte-seconds taken by all map tasks=21497856 Map-Reduce Framework Map input records=2000000 Map output records=2000000 Input split bytes=109 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=133 CPU time spent (ms)=10080 Physical memory (bytes) snapshot=266956800 Virtual memory (bytes) snapshot=4167409664 Total committed heap usage (bytes)=308805632 File Input Format Counters Bytes Read=0 File Output Format Counters Bytes Written=56555635 2016-08-27 16:35:22,913 INFO [main] test.IntegrationTestBigLinkedList$Generator: Running Generator with numMappers=1, numNodes=2000000 2016-08-27 16:35:22,915 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x67ecc6e5 connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:35:22,915 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@69f619b0 2016-08-27 16:35:22,919 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase00.ncku.edu.tw/192.168.103.142:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:35:22,922 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase00.ncku.edu.tw/192.168.103.142:2181, initiating session 2016-08-27 16:35:22,927 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase00.ncku.edu.tw/192.168.103.142:2181, sessionid = 0x56cbb36424004a, negotiated timeout = 90000 2016-08-27 16:35:23,191 INFO [main] test.IntegrationTestBigLinkedList$Generator: Number of live regionservers: 6, pre-splitting table into 18 regions (default regions per server: 3) 2016-08-27 16:35:25,515 INFO [main] client.HBaseAdmin: Created IntegrationTestBigLinkedList 2016-08-27 16:35:25,515 INFO [main] client.ConnectionManager$HConnectionImplementation: Closing master protocol: MasterService 2016-08-27 16:35:25,517 INFO [main] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x56cbb36424004a 2016-08-27 16:35:25,535 INFO [main] zookeeper.ZooKeeper: Session: 0x56cbb36424004a closed 2016-08-27 16:35:25,535 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down 2016-08-27 16:35:25,638 INFO [main] client.RMProxy: Connecting to ResourceManager at hbase00/192.168.103.142:8032 2016-08-27 16:35:26,548 INFO [main] input.FileInputFormat: Total input paths to process : 1 2016-08-27 16:35:26,635 INFO [main] mapreduce.JobSubmitter: number of splits:1 2016-08-27 16:35:26,700 INFO [main] mapreduce.JobSubmitter: Submitting tokens for job: job_1472292515371_0047 2016-08-27 16:35:26,797 INFO [main] impl.YarnClientImpl: Submitted application application_1472292515371_0047 2016-08-27 16:35:26,803 INFO [main] mapreduce.Job: The url to track the job: http://hbase00:8088/proxy/application_1472292515371_0047/ 2016-08-27 16:35:26,803 INFO [main] mapreduce.Job: Running job: job_1472292515371_0047 2016-08-27 16:35:34,090 INFO [main] mapreduce.Job: Job job_1472292515371_0047 running in uber mode : false 2016-08-27 16:35:34,091 INFO [main] mapreduce.Job: map 0% reduce 0% 2016-08-27 16:35:46,296 INFO [main] mapreduce.Job: map 50% reduce 0% 2016-08-27 16:36:05,927 INFO [Thread-4] actions.Action: Moving 13 regions from 4 servers to 2 different servers 2016-08-27 16:36:08,642 INFO [Thread-4] actions.Action: Killing region server:hbase01.ncku.edu.tw,16020,1472296349336 2016-08-27 16:36:08,642 INFO [Thread-4] hbase.HBaseCluster: Aborting RS: hbase01.ncku.edu.tw,16020,1472296349336 2016-08-27 16:36:08,645 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase01.ncku.edu.tw 2016-08-27 16:36:08,646 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase01.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL"] 2016-08-27 16:36:09,141 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:36:09,142 INFO [Thread-4] hbase.HBaseCluster: Waiting for service: regionserver to stop: hbase01.ncku.edu.tw,16020,1472296349336 2016-08-27 16:36:09,142 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:hbase01.ncku.edu.tw 2016-08-27 16:36:09,142 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase01.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2"] 2016-08-27 16:36:09,570 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:36:09,577 INFO [Thread-4] actions.Action: Killed region server:hbase01.ncku.edu.tw,16020,1472296349336. Reported num of rs:6 2016-08-27 16:36:10,544 INFO [main] mapreduce.Job: map 100% reduce 0% 2016-08-27 16:36:11,597 ERROR [Thread-4] actions.Action: Balancer didn't succeed 2016-08-27 16:36:16,598 INFO [Thread-4] actions.Action: Starting region server:hbase01.ncku.edu.tw 2016-08-27 16:36:16,598 INFO [Thread-4] hbase.HBaseCluster: Starting RS on: hbase01.ncku.edu.tw 2016-08-27 16:36:16,600 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: /root/hbase/bin/../bin/hbase-daemon.sh start regionserver , hostname:hbase01.ncku.edu.tw 2016-08-27 16:36:16,600 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase01.ncku.edu.tw "sudo -u root /root/hbase/bin/../bin/hbase-daemon.sh start regionserver"] 2016-08-27 16:36:18,047 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /root/hbase/bin/../logs/hbase-root-regionserver-hbase01.ncku.edu.tw.out Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 2016-08-27 16:37:18,095 WARN [Thread-4] policies.Policy: Exception occured during performing action: java.io.IOException: did timeout 60000ms waiting for region server to start: hbase01.ncku.edu.tw at org.apache.hadoop.hbase.HBaseCluster.waitForRegionServerToStart(HBaseCluster.java:154) at org.apache.hadoop.hbase.chaos.actions.Action.startRs(Action.java:157) at org.apache.hadoop.hbase.chaos.actions.UnbalanceKillAndRebalanceAction.perform(UnbalanceKillAndRebalanceAction.java:82) at org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy.runOneIteration(PeriodicRandomActionPolicy.java:59) at org.apache.hadoop.hbase.chaos.policies.PeriodicPolicy.run(PeriodicPolicy.java:41) at java.lang.Thread.run(Thread.java:745) 2016-08-27 16:37:18,101 INFO [Thread-4] actions.Action: Moving 14 regions from 3 servers to 2 different servers 2016-08-27 16:37:21,197 INFO [Thread-4] actions.Action: Killing region server:hbase03.ncku.edu.tw,16020,1472296349784 2016-08-27 16:37:21,198 INFO [Thread-4] hbase.HBaseCluster: Aborting RS: hbase03.ncku.edu.tw,16020,1472296349784 2016-08-27 16:37:21,198 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase03.ncku.edu.tw 2016-08-27 16:37:21,198 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase03.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL"] 2016-08-27 16:37:21,701 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:37:21,701 INFO [Thread-4] hbase.HBaseCluster: Waiting for service: regionserver to stop: hbase03.ncku.edu.tw,16020,1472296349784 2016-08-27 16:37:21,701 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:hbase03.ncku.edu.tw 2016-08-27 16:37:21,701 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase03.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2"] 2016-08-27 16:37:22,162 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:37:22,172 INFO [Thread-4] actions.Action: Killed region server:hbase03.ncku.edu.tw,16020,1472296349784. Reported num of rs:5 2016-08-27 16:37:24,176 ERROR [Thread-4] actions.Action: Balancer didn't succeed 2016-08-27 16:37:29,177 INFO [Thread-4] actions.Action: Starting region server:hbase03.ncku.edu.tw 2016-08-27 16:37:29,177 INFO [Thread-4] hbase.HBaseCluster: Starting RS on: hbase03.ncku.edu.tw 2016-08-27 16:37:29,178 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: /root/hbase/bin/../bin/hbase-daemon.sh start regionserver , hostname:hbase03.ncku.edu.tw 2016-08-27 16:37:29,178 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase03.ncku.edu.tw "sudo -u root /root/hbase/bin/../bin/hbase-daemon.sh start regionserver"] 2016-08-27 16:37:30,653 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /root/hbase/bin/../logs/hbase-root-regionserver-hbase03.ncku.edu.tw.out Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 2016-08-27 16:38:20,638 INFO [main] mapreduce.Job: Job job_1472292515371_0047 completed successfully 2016-08-27 16:38:20,702 INFO [main] mapreduce.Job: Counters: 30 File System Counters FILE: Number of bytes read=0 FILE: Number of bytes written=130805 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=56555794 HDFS: Number of bytes written=0 HDFS: Number of read operations=3 HDFS: Number of large read operations=0 HDFS: Number of write operations=0 Job Counters Launched map tasks=1 Rack-local map tasks=1 Total time spent by all maps in occupied slots (ms)=327856 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=163928 Total vcore-seconds taken by all map tasks=327856 Total megabyte-seconds taken by all map tasks=335724544 Map-Reduce Framework Map input records=2000000 Map output records=0 Input split bytes=159 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=2008 CPU time spent (ms)=164150 Physical memory (bytes) snapshot=995594240 Virtual memory (bytes) snapshot=4216053760 Total committed heap usage (bytes)=800587776 File Input Format Counters Bytes Read=56555635 File Output Format Counters Bytes Written=0 2016-08-27 16:38:20,707 INFO [main] test.IntegrationTestBigLinkedList$Verify: Running Verify with outputDir=/tmp/it_16224/46f895d7-5673-4862-a7ab-268607b3ccf8, numReducers=1 2016-08-27 16:38:20,836 INFO [main] client.RMProxy: Connecting to ResourceManager at hbase00/192.168.103.142:8032 2016-08-27 16:38:21,566 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x64f25d57 connecting to ZooKeeper ensemble=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 2016-08-27 16:38:21,566 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hbase00.ncku.edu.tw:2181,hbase01.ncku.edu.tw:2181,hbase02.ncku.edu.tw:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@63ce2e87 2016-08-27 16:38:21,568 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Opening socket connection to server hbase00.ncku.edu.tw/192.168.103.142:2181. Will not attempt to authenticate using SASL (unknown error) 2016-08-27 16:38:21,569 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Socket connection established to hbase00.ncku.edu.tw/192.168.103.142:2181, initiating session 2016-08-27 16:38:21,573 INFO [main-SendThread(hbase00.ncku.edu.tw:2181)] zookeeper.ClientCnxn: Session establishment complete on server hbase00.ncku.edu.tw/192.168.103.142:2181, sessionid = 0x56cbb36424004d, negotiated timeout = 90000 2016-08-27 16:38:21,578 INFO [main] util.RegionSizeCalculator: Calculating region sizes for table "IntegrationTestBigLinkedList". 2016-08-27 16:38:21,914 INFO [htable-pool3-t1] client.ScannerCallable: Open scanner=9931 for scan={"timeRange":[0,9223372036854775807],"batch":-1,"startRow":"IntegrationTestBigLinkedList,,00000000000000","stopRow":"","loadColumnFamiliesOnDemand":null,"totalColumns":1,"cacheBlocks":true,"families":{"info":["ALL"]},"maxResultSize":2097152,"maxVersions":1,"caching":100} on region region=hbase:meta,,1.1588230740, hostname=hbase04.ncku.edu.tw,16020,1472296349779, seqNum=0 2016-08-27 16:38:21,982 INFO [htable-pool4-t1] client.ScannerCallable: Open scanner=9932 for scan={"timeRange":[0,9223372036854775807],"batch":-1,"startRow":"IntegrationTestBigLinkedList,,00000000000000","stopRow":"","loadColumnFamiliesOnDemand":null,"totalColumns":1,"cacheBlocks":true,"families":{"info":["ALL"]},"maxResultSize":2097152,"maxVersions":1,"caching":100} on region region=hbase:meta,,1.1588230740, hostname=hbase04.ncku.edu.tw,16020,1472296349779, seqNum=0 2016-08-27 16:38:22,126 INFO [main] client.ConnectionManager$HConnectionImplementation: Closing master protocol: MasterService 2016-08-27 16:38:22,129 INFO [main] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x56cbb36424004d 2016-08-27 16:38:22,134 INFO [main] zookeeper.ZooKeeper: Session: 0x56cbb36424004d closed 2016-08-27 16:38:22,134 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down 2016-08-27 16:38:22,232 INFO [main] mapreduce.JobSubmitter: number of splits:18 2016-08-27 16:38:22,281 INFO [main] mapreduce.JobSubmitter: Submitting tokens for job: job_1472292515371_0048 2016-08-27 16:38:22,368 INFO [main] impl.YarnClientImpl: Submitted application application_1472292515371_0048 2016-08-27 16:38:22,374 INFO [main] mapreduce.Job: The url to track the job: http://hbase00:8088/proxy/application_1472292515371_0048/ 2016-08-27 16:38:22,374 INFO [main] mapreduce.Job: Running job: job_1472292515371_0048 2016-08-27 16:38:29,600 INFO [main] mapreduce.Job: Job job_1472292515371_0048 running in uber mode : false 2016-08-27 16:38:29,600 INFO [main] mapreduce.Job: map 0% reduce 0% 2016-08-27 16:38:30,695 WARN [Thread-4] policies.Policy: Exception occured during performing action: java.io.IOException: did timeout 60000ms waiting for region server to start: hbase03.ncku.edu.tw at org.apache.hadoop.hbase.HBaseCluster.waitForRegionServerToStart(HBaseCluster.java:154) at org.apache.hadoop.hbase.chaos.actions.Action.startRs(Action.java:157) at org.apache.hadoop.hbase.chaos.actions.UnbalanceKillAndRebalanceAction.perform(UnbalanceKillAndRebalanceAction.java:82) at org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy.runOneIteration(PeriodicRandomActionPolicy.java:59) at org.apache.hadoop.hbase.chaos.policies.PeriodicPolicy.run(PeriodicPolicy.java:41) at java.lang.Thread.run(Thread.java:745) 2016-08-27 16:38:30,700 INFO [Thread-4] actions.Action: Moving 13 regions from 2 servers to 2 different servers 2016-08-27 16:38:36,684 INFO [Thread-4] actions.Action: Killing region server:hbase02.ncku.edu.tw,16020,1472296349810 2016-08-27 16:38:36,684 INFO [Thread-4] hbase.HBaseCluster: Aborting RS: hbase02.ncku.edu.tw,16020,1472296349810 2016-08-27 16:38:36,685 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL , hostname:hbase02.ncku.edu.tw 2016-08-27 16:38:36,685 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase02.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s SIGKILL"] 2016-08-27 16:38:37,217 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:38:37,217 INFO [Thread-4] hbase.HBaseCluster: Waiting for service: regionserver to stop: hbase02.ncku.edu.tw,16020,1472296349810 2016-08-27 16:38:37,218 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:hbase02.ncku.edu.tw 2016-08-27 16:38:37,218 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase02.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2"] 2016-08-27 16:38:37,684 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:38:37,690 INFO [Thread-4] actions.Action: Killed region server:hbase02.ncku.edu.tw,16020,1472296349810. Reported num of rs:4 2016-08-27 16:38:39,694 ERROR [Thread-4] actions.Action: Balancer didn't succeed 2016-08-27 16:38:43,805 INFO [main] mapreduce.Job: map 11% reduce 0% 2016-08-27 16:38:44,694 INFO [Thread-4] actions.Action: Starting region server:hbase02.ncku.edu.tw 2016-08-27 16:38:44,695 INFO [Thread-4] hbase.HBaseCluster: Starting RS on: hbase02.ncku.edu.tw 2016-08-27 16:38:44,695 INFO [Thread-4] hbase.HBaseClusterManager: Executing remote command: /root/hbase/bin/../bin/hbase-daemon.sh start regionserver , hostname:hbase02.ncku.edu.tw 2016-08-27 16:38:44,696 INFO [Thread-4] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase02.ncku.edu.tw "sudo -u root /root/hbase/bin/../bin/hbase-daemon.sh start regionserver"] 2016-08-27 16:38:44,815 INFO [main] mapreduce.Job: map 33% reduce 0% 2016-08-27 16:38:45,844 INFO [main] mapreduce.Job: map 48% reduce 0% 2016-08-27 16:38:46,152 INFO [Thread-4] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /root/hbase/bin/../logs/hbase-root-regionserver-hbase02.ncku.edu.tw.out Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 2016-08-27 16:38:46,854 INFO [main] mapreduce.Job: map 67% reduce 0% 2016-08-27 16:38:51,899 INFO [main] mapreduce.Job: map 72% reduce 0% 2016-08-27 16:38:53,915 INFO [main] mapreduce.Job: map 83% reduce 0% 2016-08-27 16:38:55,932 INFO [main] mapreduce.Job: map 83% reduce 28% 2016-08-27 16:39:00,977 INFO [main] mapreduce.Job: map 89% reduce 28% 2016-08-27 16:39:01,989 INFO [main] mapreduce.Job: map 100% reduce 30% 2016-08-27 16:39:05,015 INFO [main] mapreduce.Job: map 100% reduce 57% 2016-08-27 16:39:08,041 INFO [main] mapreduce.Job: map 100% reduce 79% 2016-08-27 16:39:11,064 INFO [main] mapreduce.Job: map 100% reduce 100% 2016-08-27 16:39:11,077 INFO [main] mapreduce.Job: Job job_1472292515371_0048 completed successfully 2016-08-27 16:39:11,143 INFO [main] mapreduce.Job: Counters: 51 File System Counters FILE: Number of bytes read=138000006 FILE: Number of bytes written=278470124 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=3868 HDFS: Number of bytes written=96 HDFS: Number of read operations=39 HDFS: Number of large read operations=0 HDFS: Number of write operations=2 Job Counters Launched map tasks=18 Launched reduce tasks=1 Data-local map tasks=14 Rack-local map tasks=4 Total time spent by all maps in occupied slots (ms)=629912 Total time spent by all reduces in occupied slots (ms)=48710 Total time spent by all map tasks (ms)=314956 Total time spent by all reduce tasks (ms)=24355 Total vcore-seconds taken by all map tasks=629912 Total vcore-seconds taken by all reduce tasks=48710 Total megabyte-seconds taken by all map tasks=645029888 Total megabyte-seconds taken by all reduce tasks=49879040 Map-Reduce Framework Map input records=2000000 Map output records=4000000 Map output bytes=130000000 Map output materialized bytes=138000108 Input split bytes=3868 Combine input records=0 Combine output records=0 Reduce input groups=2000000 Reduce shuffle bytes=138000108 Reduce input records=4000000 Reduce output records=0 Spilled Records=8000000 Shuffled Maps =18 Failed Shuffles=0 Merged Map outputs=18 GC time elapsed (ms)=4330 CPU time spent (ms)=237810 Physical memory (bytes) snapshot=8757809152 Virtual memory (bytes) snapshot=80095133696 Total committed heap usage (bytes)=6725566464 Shuffle Errors BAD_ID=0 CONNECTION=0 IO_ERROR=0 WRONG_LENGTH=0 WRONG_MAP=0 WRONG_REDUCE=0 org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList$Verify$Counts REFERENCED=2000000 File Input Format Counters Bytes Read=0 File Output Format Counters Bytes Written=96 2016-08-27 16:39:11,160 INFO [main] test.IntegrationTestBigLinkedList$Loop: Verify finished with success. Total nodes=2000000 2016-08-27 16:39:46,226 WARN [Thread-4] policies.Policy: Exception occured during performing action: java.io.IOException: did timeout 60000ms waiting for region server to start: hbase02.ncku.edu.tw at org.apache.hadoop.hbase.HBaseCluster.waitForRegionServerToStart(HBaseCluster.java:154) at org.apache.hadoop.hbase.chaos.actions.Action.startRs(Action.java:157) at org.apache.hadoop.hbase.chaos.actions.UnbalanceKillAndRebalanceAction.perform(UnbalanceKillAndRebalanceAction.java:82) at org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy.runOneIteration(PeriodicRandomActionPolicy.java:59) at org.apache.hadoop.hbase.chaos.policies.PeriodicPolicy.run(PeriodicPolicy.java:41) at java.lang.Thread.run(Thread.java:745) 2016-08-27 16:39:46,230 INFO [main] hbase.HBaseCluster: Restoring cluster - started 2016-08-27 16:39:46,235 INFO [main] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:hbase01.ncku.edu.tw 2016-08-27 16:39:46,235 INFO [main] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase01.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2"] 2016-08-27 16:39:46,665 INFO [main] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:39:46,665 INFO [main] hbase.HBaseCluster: Restoring cluster - starting initial region server: hbase01.ncku.edu.tw:16020 2016-08-27 16:39:46,666 INFO [main] hbase.HBaseCluster: Starting RS on: hbase01.ncku.edu.tw 2016-08-27 16:39:46,666 INFO [main] hbase.HBaseClusterManager: Executing remote command: /root/hbase/bin/../bin/hbase-daemon.sh start regionserver , hostname:hbase01.ncku.edu.tw 2016-08-27 16:39:46,666 INFO [main] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase01.ncku.edu.tw "sudo -u root /root/hbase/bin/../bin/hbase-daemon.sh start regionserver"] 2016-08-27 16:39:48,091 INFO [main] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /root/hbase/bin/../logs/hbase-root-regionserver-hbase01.ncku.edu.tw.out Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 2016-08-27 16:39:48,091 INFO [main] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:hbase02.ncku.edu.tw 2016-08-27 16:39:48,092 INFO [main] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase02.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2"] 2016-08-27 16:39:48,518 INFO [main] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:39:48,518 INFO [main] hbase.HBaseCluster: Restoring cluster - starting initial region server: hbase02.ncku.edu.tw:16020 2016-08-27 16:39:48,518 INFO [main] hbase.HBaseCluster: Starting RS on: hbase02.ncku.edu.tw 2016-08-27 16:39:48,519 INFO [main] hbase.HBaseClusterManager: Executing remote command: /root/hbase/bin/../bin/hbase-daemon.sh start regionserver , hostname:hbase02.ncku.edu.tw 2016-08-27 16:39:48,519 INFO [main] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase02.ncku.edu.tw "sudo -u root /root/hbase/bin/../bin/hbase-daemon.sh start regionserver"] 2016-08-27 16:39:49,936 INFO [main] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /root/hbase/bin/../logs/hbase-root-regionserver-hbase02.ncku.edu.tw.out Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 2016-08-27 16:39:49,937 INFO [main] hbase.HBaseClusterManager: Executing remote command: ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2 , hostname:hbase03.ncku.edu.tw 2016-08-27 16:39:49,937 INFO [main] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase03.ncku.edu.tw "sudo -u root ps ux | grep proc_regionserver | grep -v grep | tr -s ' ' | cut -d ' ' -f2"] 2016-08-27 16:39:50,359 INFO [main] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output: 2016-08-27 16:39:50,360 INFO [main] hbase.HBaseCluster: Restoring cluster - starting initial region server: hbase03.ncku.edu.tw:16020 2016-08-27 16:39:50,360 INFO [main] hbase.HBaseCluster: Starting RS on: hbase03.ncku.edu.tw 2016-08-27 16:39:50,360 INFO [main] hbase.HBaseClusterManager: Executing remote command: /root/hbase/bin/../bin/hbase-daemon.sh start regionserver , hostname:hbase03.ncku.edu.tw 2016-08-27 16:39:50,360 INFO [main] util.Shell: Executing full command [/usr/bin/ssh -tt root@hbase03.ncku.edu.tw "sudo -u root /root/hbase/bin/../bin/hbase-daemon.sh start regionserver"] 2016-08-27 16:39:51,764 INFO [main] hbase.HBaseClusterManager: Executed remote command, exit code:0 , output:starting regionserver, logging to /root/hbase/bin/../logs/hbase-root-regionserver-hbase03.ncku.edu.tw.out Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=128m; support was removed in 8.0 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0 2016-08-27 16:39:51,765 INFO [main] hbase.HBaseCluster: Added new HBaseAdmin 2016-08-27 16:39:51,765 INFO [main] hbase.HBaseCluster: Restoring cluster - done [root@compiler00 hbase]#