org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.187.190:57785,127.7.187.188:47280,127.7.187.189:45658) has no leader. at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:279) at org.apache.kudu.client.ConnectToCluster.access$100(ConnectToCluster.java:47) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:323) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:312) at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:247) at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:294) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:269) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.messageReceived(Connection.java:391) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) /127.3.230.189:53922] using SASL/PLAIN 04:43:52.000 [DEBUG - New I/O worker #2] (Negotiator.java:764) Authenticated connection [id: 0x5c1d4812, /127.3.230.188:33264 => /127.3.230.188:43675] using SASL/PLAIN 04:43:52.001 [DEBUG - New I/O worker #1] (Negotiator.java:764) Authenticated connection [id: 0xa5c326e9, /127.3.230.190:46168 => /127.3.230.190:46041] using SASL/PLAIN 04:43:52.027 [DEBUG - New I/O worker #3] (AsyncKuduClient.java:1985) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 04:43:52.028 [DEBUG - New I/O worker #3] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=315), Traces: [0ms] querying master, [31ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [41ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [41ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [296ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [296ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [296ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response OK, null) after lookup 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.035331 4115 catalog_manager.cc:1366] Servicing CreateTable request from {username='jenkins'} at 127.3.230.189:42914: 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "TestMasterFailover-killBefore=OPEN_TABLE" 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) schema { 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "key" 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: true 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column1_i" 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column2_i" 04:43:52.036 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=retry RPC@811419037 returned Deferred@943753109(state=PENDING, result=null, callback=(continuation of Deferred@531124436 after retry RPC@811419037), errback=(continuation of Deferred@531124436 after retry RPC@811419037)), so the following Deferred is getting paused: Deferred@531124436(state=PAUSED, result=Deferred@943753109, callback=org.apache.kudu.client.AsyncKuduClient$1@17f713ee -> org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20 -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:43:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column3_s" 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: STRING 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: true 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) encoding: DICT_ENCODING 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) compression: LZ4 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 4096 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column4_b" 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: BOOL 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:43:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) partition_schema { 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) range_schema { 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "key" 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:43:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.035424 4115 catalog_manager.cc:1504] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table TestMasterFailover-killBefore=OPEN_TABLE in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 04:43:52.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.041594 4277 tablet_service.cc:759] Processing CreateTablet for tablet dc16229ae54d418c82f07b64e4faf2ea (table=TestMasterFailover-killBefore=OPEN_TABLE [id=c5cc0f2dcd0b4747a1f944fd14bcdbf9]), partition=RANGE (key) PARTITION UNBOUNDED 04:43:52.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.041676 4277 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet dc16229ae54d418c82f07b64e4faf2ea. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:52.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.043505 4277 ts_tablet_manager.cc:1173] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Registered tablet (data state: TABLET_DATA_READY) 04:43:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.045142 4419 tablet_service.cc:759] Processing CreateTablet for tablet dc16229ae54d418c82f07b64e4faf2ea (table=TestMasterFailover-killBefore=OPEN_TABLE [id=c5cc0f2dcd0b4747a1f944fd14bcdbf9]), partition=RANGE (key) PARTITION UNBOUNDED 04:43:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.045240 4419 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet dc16229ae54d418c82f07b64e4faf2ea. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:52.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.046083 4546 tablet_service.cc:759] Processing CreateTablet for tablet dc16229ae54d418c82f07b64e4faf2ea (table=TestMasterFailover-killBefore=OPEN_TABLE [id=c5cc0f2dcd0b4747a1f944fd14bcdbf9]), partition=RANGE (key) PARTITION UNBOUNDED 04:43:52.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.046150 4546 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet dc16229ae54d418c82f07b64e4faf2ea. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:52.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.046949 4419 ts_tablet_manager.cc:1173] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Registered tablet (data state: TABLET_DATA_READY) 04:43:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.047276 4661 ts_tablet_manager.cc:1028] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Bootstrapping tablet 04:43:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.047317 4662 ts_tablet_manager.cc:1028] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Bootstrapping tablet 04:43:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.047349 4662 tablet_bootstrap.cc:438] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Bootstrap starting. 04:43:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.047600 4546 ts_tablet_manager.cc:1173] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Registered tablet (data state: TABLET_DATA_READY) 04:43:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.047808 4661 tablet_bootstrap.cc:438] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Bootstrap starting. 04:43:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.048053 4662 tablet_bootstrap.cc:592] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: No blocks or log segments found. Creating new log. 04:43:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.048228 4661 tablet_bootstrap.cc:592] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: No blocks or log segments found. Creating new log. 04:43:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.048535 4662 log.cc:525] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Log is configured to *not* fsync() on all Append() calls 04:43:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.048563 4661 log.cc:525] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Log is configured to *not* fsync() on all Append() calls 04:43:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.048804 4663 ts_tablet_manager.cc:1028] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Bootstrapping tablet 04:43:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.048964 4663 tablet_bootstrap.cc:438] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Bootstrap starting. 04:43:52.049 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@17f713ee@402068462 returned Deferred@1906083633(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2cef5726 -> (continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$1@17f713ee@402068462), errback=passthrough -> (continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$1@17f713ee@402068462)), so the following Deferred is getting paused: Deferred@531124436(state=PAUSED, result=Deferred@1906083633, callback=org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 04:43:52.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.049228 4661 tablet_bootstrap.cc:438] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: No bootstrap required, opened a new log 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.049258 4661 ts_tablet_manager.cc:1045] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.000s 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.049530 4662 tablet_bootstrap.cc:438] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: No bootstrap required, opened a new log 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.049556 4662 ts_tablet_manager.cc:1045] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.000s 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050618 4661 raft_consensus.cc:325] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050635 4661 raft_consensus.cc:351] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050644 4661 raft_consensus.cc:630] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e525d66c3444766a62305b33618b74d, State: Initialized, Role: FOLLOWER 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050704 4661 consensus_queue.cc:228] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050799 4661 ts_tablet_manager.cc:1073] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050832 4341 heartbeater.cc:467] Master 127.3.230.189:53922 was elected leader, sending a full tablet report... 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050879 4662 raft_consensus.cc:325] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050894 4662 raft_consensus.cc:351] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050901 4662 raft_consensus.cc:630] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5f67e058ce8e485998eb40d421b840f1, State: Initialized, Role: FOLLOWER 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.050957 4662 consensus_queue.cc:228] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.051051 4662 ts_tablet_manager.cc:1073] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s 04:43:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.052332 4663 tablet_bootstrap.cc:592] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: No blocks or log segments found. Creating new log. 04:43:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.052700 4663 log.cc:525] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Log is configured to *not* fsync() on all Append() calls 04:43:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.054103 4663 tablet_bootstrap.cc:438] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: No bootstrap required, opened a new log 04:43:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.054289 4663 ts_tablet_manager.cc:1045] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.000s 04:43:52.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.055722 4663 raft_consensus.cc:325] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.055927 4663 raft_consensus.cc:351] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.056072 4663 raft_consensus.cc:630] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0257d621b4a5457098a36cfdbb8f329d, State: Initialized, Role: FOLLOWER 04:43:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.056329 4663 consensus_queue.cc:228] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.056804 4663 ts_tablet_manager.cc:1073] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s 04:43:52.057 [DEBUG - New I/O worker #3] (AsyncKuduClient.java:737) Opened table c5cc0f2dcd0b4747a1f944fd14bcdbf9 04:43:52.058 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456 returned Deferred@1023807110(state=PENDING, result=null, callback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456), errback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456)), so the following Deferred is getting paused: Deferred@531124436(state=PAUSED, result=Deferred@1023807110, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 04:43:52.067 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767 returned Deferred@1023807110(state=PENDING, result=null, callback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456) -> (continuation of Deferred@707149272 after org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767), errback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456) -> (continuation of Deferred@707149272 after org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767)), so the following Deferred is getting paused: Deferred@707149272(state=PAUSED, result=Deferred@1023807110, callback=, errback=) 04:43:52.083 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@3780d263@931189347 returned Deferred@1023807110(state=PENDING, result=null, callback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456) -> (continuation of Deferred@707149272 after org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767) -> (continuation of Deferred@989825067 after org.apache.kudu.client.AsyncKuduClient$13@3780d263@931189347), errback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456) -> (continuation of Deferred@707149272 after org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767) -> (continuation of Deferred@989825067 after org.apache.kudu.client.AsyncKuduClient$13@3780d263@931189347)), so the following Deferred is getting paused: Deferred@989825067(state=PAUSED, result=Deferred@1023807110, callback=, errback=) 04:43:52.104 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@60605a13@1616927251 returned Deferred@1023807110(state=PENDING, result=null, callback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456) -> (continuation of Deferred@707149272 after org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767) -> (continuation of Deferred@989825067 after org.apache.kudu.client.AsyncKuduClient$13@3780d263@931189347) -> (continuation of Deferred@2066969402 after org.apache.kudu.client.AsyncKuduClient$13@60605a13@1616927251), errback=(continuation of Deferred@531124436 after org.apache.kudu.client.AsyncKuduClient$2@4d5ccd20@1297927456) -> (continuation of Deferred@707149272 after org.apache.kudu.client.AsyncKuduClient$13@6b063867@1795569767) -> (continuation of Deferred@989825067 after org.apache.kudu.client.AsyncKuduClient$13@3780d263@931189347) -> (continuation of Deferred@2066969402 after org.apache.kudu.client.AsyncKuduClient$13@60605a13@1616927251)), so the following Deferred is getting paused: Deferred@2066969402(state=PAUSED, result=Deferred@1023807110, callback=, errback=) 04:43:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.106977 4669 raft_consensus.cc:457] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:43:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.107017 4669 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Snoozing failure detection for 1.601s (starting election) 04:43:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.107096 4669 raft_consensus.cc:479] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.107434 4669 leader_election.cc:251] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 6e525d66c3444766a62305b33618b74d (127.3.230.129:51896) 04:43:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.107620 4669 leader_election.cc:251] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 5f67e058ce8e485998eb40d421b840f1 (127.3.230.130:55584) 04:43:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110350 4297 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "dc16229ae54d418c82f07b64e4faf2ea" candidate_uuid: "0257d621b4a5457098a36cfdbb8f329d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e525d66c3444766a62305b33618b74d" is_pre_election: true 04:43:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110383 4297 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Snoozing failure detection for 2.098s (vote granted) 04:43:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110393 4297 raft_consensus.cc:2255] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0257d621b4a5457098a36cfdbb8f329d in term 0. 04:43:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110559 4498 leader_election.cc:387] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 pre-election: Vote granted by peer 6e525d66c3444766a62305b33618b74d (127.3.230.129:51896) 04:43:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110570 4498 leader_election.cc:278] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 04:43:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110874 4669 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Snoozing failure detection for 1.906s (election complete) 04:43:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110890 4669 raft_consensus.cc:2549] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Leader pre-election won for term 1 04:43:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110898 4669 raft_consensus.cc:457] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:43:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110905 4669 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Snoozing failure detection for 1.982s (starting election) 04:43:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.110914 4669 raft_consensus.cc:2794] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 0 FOLLOWER]: Advancing to term 1 04:43:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.111559 4669 raft_consensus.cc:479] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.111623 4669 leader_election.cc:251] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 election: Requesting vote from peer 6e525d66c3444766a62305b33618b74d (127.3.230.129:51896) 04:43:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.111637 4669 leader_election.cc:251] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 election: Requesting vote from peer 5f67e058ce8e485998eb40d421b840f1 (127.3.230.130:55584) 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112598 4297 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "dc16229ae54d418c82f07b64e4faf2ea" candidate_uuid: "0257d621b4a5457098a36cfdbb8f329d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e525d66c3444766a62305b33618b74d" 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112618 4297 raft_consensus.cc:2794] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 0 FOLLOWER]: Advancing to term 1 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112632 4297 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Snoozing failure detection for 2.067s (vote granted) 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112730 4439 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "dc16229ae54d418c82f07b64e4faf2ea" candidate_uuid: "0257d621b4a5457098a36cfdbb8f329d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5f67e058ce8e485998eb40d421b840f1" is_pre_election: true 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112756 4439 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Snoozing failure detection for 1.905s (vote granted) 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112766 4439 raft_consensus.cc:2255] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0257d621b4a5457098a36cfdbb8f329d in term 0. 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112819 4438 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "dc16229ae54d418c82f07b64e4faf2ea" candidate_uuid: "0257d621b4a5457098a36cfdbb8f329d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5f67e058ce8e485998eb40d421b840f1" 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112830 4438 raft_consensus.cc:2794] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 0 FOLLOWER]: Advancing to term 1 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.112841 4438 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Snoozing failure detection for 1.911s (vote granted) 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113157 4297 raft_consensus.cc:2255] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0257d621b4a5457098a36cfdbb8f329d in term 1. 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113343 4438 raft_consensus.cc:2255] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0257d621b4a5457098a36cfdbb8f329d in term 1. 04:43:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113499 4498 leader_election.cc:387] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 pre-election: Vote granted by peer 5f67e058ce8e485998eb40d421b840f1 (127.3.230.130:55584) 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113543 4498 leader_election.cc:387] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 election: Vote granted by peer 6e525d66c3444766a62305b33618b74d (127.3.230.129:51896) 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113550 4498 leader_election.cc:278] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113610 4498 leader_election.cc:387] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [CANDIDATE]: Term 1 election: Vote granted by peer 5f67e058ce8e485998eb40d421b840f1 (127.3.230.130:55584) 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113642 4669 raft_consensus.cc:2742] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Snoozing failure detection for 1.910s (election complete) 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113651 4669 raft_consensus.cc:2549] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 1 FOLLOWER]: Leader election won for term 1 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113715 4669 raft_consensus.cc:595] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [term 1 LEADER]: Becoming Leader. State: Replica: 0257d621b4a5457098a36cfdbb8f329d, State: Running, Role: LEADER 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.113770 4669 consensus_queue.cc:205] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } } 04:43:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.114413 4115 catalog_manager.cc:3758] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d reported cstate change: term changed from 0 to 1, leader changed from to 0257d621b4a5457098a36cfdbb8f329d (127.3.230.131). New cstate: current_term: 1 leader_uuid: "0257d621b4a5457098a36cfdbb8f329d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0257d621b4a5457098a36cfdbb8f329d" member_type: VOTER last_known_addr { host: "127.3.230.131" port: 47182 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 } health_report { overall_health: UNKNOWN } } } 04:43:52.125 [INFO - Time-limited test] (MiniKuduCluster.java:320) Killing master server 127.3.230.189:53922 04:43:52.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:163) Request: stop_daemon { id { type: MASTER index: 1 } } 04:43:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.126689 3994 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 4089 04:43:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.129189 4361 connection.cc:512] client connection to 127.3.230.189:53922 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.189:53922): Connection reset by peer (error 104) 04:43:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.129189 4231 connection.cc:512] client connection to 127.3.230.189:53922 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.189:53922): Connection reset by peer (error 104) 04:43:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.129308 4500 connection.cc:512] client connection to 127.3.230.189:53922 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.189:53922): Connection reset by peer (error 104) 04:43:52.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:173) Response: 04:43:52.130 [DEBUG - New I/O worker #3] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state READY due to: connection disconnected 04:43:52.130 [INFO - New I/O worker #3] (AsyncKuduClient.java:1837) Invalidating location master-127.3.230.189:53922(127.3.230.189:53922) for tablet Kudu Master: connection disconnected 04:43:52.131 [INFO - New I/O worker #3] (Connection.java:435) [peer master-127.3.230.189:53922(127.3.230.189:53922)] lost connection to peer 04:43:52.145 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.145 [DEBUG - New I/O worker #4] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.145 [INFO - New I/O worker #4] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.147 [WARN - New I/O worker #4] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.148 [DEBUG - New I/O worker #4] (Deferred.java:1330) callback=retry RPC after error@704061551 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551)), so the following Deferred is getting paused: Deferred@1354430464(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.164 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.164 [DEBUG - New I/O worker #5] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.164 [INFO - New I/O worker #5] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.165 [WARN - New I/O worker #5] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.165 [DEBUG - New I/O worker #5] (Deferred.java:1330) callback=retry RPC after error@221712901 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901)), so the following Deferred is getting paused: Deferred@66121171(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.184 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.184 [DEBUG - New I/O boss #9] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.184 [INFO - New I/O boss #9] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.185 [WARN - New I/O boss #9] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.185 [DEBUG - New I/O boss #9] (Deferred.java:1330) callback=retry RPC after error@550998666 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666)), so the following Deferred is getting paused: Deferred@1075782980(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.204 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.204 [DEBUG - New I/O worker #7] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.204 [INFO - New I/O worker #7] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.205 [WARN - New I/O worker #7] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.205 [DEBUG - New I/O worker #7] (Deferred.java:1330) callback=retry RPC after error@879212831 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831)), so the following Deferred is getting paused: Deferred@983585251(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.210428 4486 tablet.cc:1728] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:52.244 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.244 [DEBUG - New I/O worker #8] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.245 [INFO - New I/O worker #8] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.245 [WARN - New I/O worker #8] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.245 [DEBUG - New I/O worker #8] (Deferred.java:1330) callback=retry RPC after error@1506904396 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396)), so the following Deferred is getting paused: Deferred@1461087238(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.248299 4343 tablet.cc:1728] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:52.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.277196 4612 tablet.cc:1728] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:52.283 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.283 [DEBUG - New I/O boss #9] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.284 [INFO - New I/O boss #9] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.284 [WARN - New I/O boss #9] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.284 [DEBUG - New I/O boss #9] (Deferred.java:1330) callback=retry RPC after error@707264731 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731)), so the following Deferred is getting paused: Deferred@2069345643(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.364 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.364 [DEBUG - New I/O worker #2] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.364 [INFO - New I/O worker #2] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.364 [WARN - New I/O worker #2] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.365 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=retry RPC after error@1311970744 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744)), so the following Deferred is getting paused: Deferred@385987481(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.503 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.504 [DEBUG - New I/O worker #3] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.504 [INFO - New I/O worker #3] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.504 [WARN - New I/O worker #3] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.504 [DEBUG - New I/O worker #3] (Deferred.java:1330) callback=retry RPC after error@1849405696 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696)), so the following Deferred is getting paused: Deferred@242333022(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.644 [DEBUG - New I/O worker #4] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.644 [INFO - New I/O worker #4] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:52.644 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.644 [WARN - New I/O worker #4] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.644 [DEBUG - New I/O worker #4] (Deferred.java:1330) callback=retry RPC after error@1088314912 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912)), so the following Deferred is getting paused: Deferred@1645517945(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.659060 4297 raft_consensus.cc:1083] T dc16229ae54d418c82f07b64e4faf2ea P 6e525d66c3444766a62305b33618b74d [term 1 FOLLOWER]: Refusing update from remote peer 0257d621b4a5457098a36cfdbb8f329d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 04:43:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.659431 4674 consensus_queue.cc:963] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6e525d66c3444766a62305b33618b74d" member_type: VOTER last_known_addr { host: "127.3.230.129" port: 51896 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:43:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.660130 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.660647 4438 raft_consensus.cc:1083] T dc16229ae54d418c82f07b64e4faf2ea P 5f67e058ce8e485998eb40d421b840f1 [term 1 FOLLOWER]: Refusing update from remote peer 0257d621b4a5457098a36cfdbb8f329d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 04:43:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:52.660934 4679 consensus_queue.cc:963] T dc16229ae54d418c82f07b64e4faf2ea P 0257d621b4a5457098a36cfdbb8f329d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5f67e058ce8e485998eb40d421b840f1" member_type: VOTER last_known_addr { host: "127.3.230.130" port: 55584 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:43:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.661226 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.661265 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.661999 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:52.662293 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:52.864 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.864 [DEBUG - New I/O boss #9] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.864 [INFO - New I/O boss #9] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.864 [WARN - New I/O boss #9] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.865 [DEBUG - New I/O boss #9] (Deferred.java:1330) callback=retry RPC after error@2071608599 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599)), so the following Deferred is getting paused: Deferred@235923358(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:52.904 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.904 [DEBUG - New I/O boss #9] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.904 [INFO - New I/O boss #9] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:52.905 [WARN - New I/O boss #9] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:52.905 [DEBUG - New I/O boss #9] (Deferred.java:1330) callback=retry RPC after error@417210416 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416)), so the following Deferred is getting paused: Deferred@853943481(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.112337 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616178 4686 raft_consensus.cc:457] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 87c8672d5f194f769df808882366f81b) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616194 4687 raft_consensus.cc:457] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 87c8672d5f194f769df808882366f81b) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616222 4686 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 2.134s (starting election) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616223 4687 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 1.921s (starting election) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616266 4686 raft_consensus.cc:479] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616273 4687 raft_consensus.cc:479] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616356 4686 leader_election.cc:251] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616363 4687 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616385 4687 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616385 4686 leader_election.cc:251] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer bc63d858698e41f2ab32791255236e90 (127.3.230.188:43675) 04:43:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616549 4187 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eea9f8bb6b784102a362e3ab27e00de9" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "bc63d858698e41f2ab32791255236e90" is_pre_election: true 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616555 4063 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bc63d858698e41f2ab32791255236e90" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "eea9f8bb6b784102a362e3ab27e00de9" is_pre_election: true 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616566 4187 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 1.641s (vote granted) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616575 4063 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 1.648s (vote granted) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616576 4187 raft_consensus.cc:2255] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eea9f8bb6b784102a362e3ab27e00de9 in term 1. 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616585 4063 raft_consensus.cc:2255] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bc63d858698e41f2ab32791255236e90 in term 1. 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.616634 4161 leader_election.cc:307] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616655 4035 leader_election.cc:387] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 pre-election: Vote granted by peer bc63d858698e41f2ab32791255236e90 (127.3.230.188:43675) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616673 4035 leader_election.cc:278] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.616690 4037 leader_election.cc:307] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616765 4160 leader_election.cc:387] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 pre-election: Vote granted by peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616775 4160 leader_election.cc:278] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616817 4687 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 1.587s (election complete) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616820 4690 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 1.776s (election complete) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616830 4690 raft_consensus.cc:2549] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 1 FOLLOWER]: Leader pre-election won for term 2 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616830 4687 raft_consensus.cc:2549] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 1 FOLLOWER]: Leader pre-election won for term 2 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616837 4687 raft_consensus.cc:457] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 87c8672d5f194f769df808882366f81b) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616837 4690 raft_consensus.cc:457] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 87c8672d5f194f769df808882366f81b) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616843 4687 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 1.635s (starting election) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616844 4690 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 1.540s (starting election) 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616849 4690 raft_consensus.cc:2794] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 1 FOLLOWER]: Advancing to term 2 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.616848 4687 raft_consensus.cc:2794] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 1 FOLLOWER]: Advancing to term 2 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617482 4690 raft_consensus.cc:479] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617482 4687 raft_consensus.cc:479] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617542 4690 leader_election.cc:251] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 election: Requesting vote from peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617542 4687 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 election: Requesting vote from peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617565 4687 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 election: Requesting vote from peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617609 4690 leader_election.cc:251] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 election: Requesting vote from peer bc63d858698e41f2ab32791255236e90 (127.3.230.188:43675) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617699 4063 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bc63d858698e41f2ab32791255236e90" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "eea9f8bb6b784102a362e3ab27e00de9" 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617718 4063 raft_consensus.cc:2185] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate bc63d858698e41f2ab32791255236e90 in current term 2: Already voted for candidate eea9f8bb6b784102a362e3ab27e00de9 in this term. 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617733 4187 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eea9f8bb6b784102a362e3ab27e00de9" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "bc63d858698e41f2ab32791255236e90" 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617748 4187 raft_consensus.cc:2185] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate eea9f8bb6b784102a362e3ab27e00de9 in current term 2: Already voted for candidate bc63d858698e41f2ab32791255236e90 in this term. 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.617764 4161 leader_election.cc:307] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.617802 4037 leader_election.cc:307] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617802 4160 leader_election.cc:401] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 election: Vote denied by peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041). Message: Invalid argument: T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate bc63d858698e41f2ab32791255236e90 in current term 2: Already voted for candidate eea9f8bb6b784102a362e3ab27e00de9 in this term. 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617813 4160 leader_election.cc:278] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617852 4687 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 1.515s (election complete) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617863 4687 raft_consensus.cc:2497] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617864 4035 leader_election.cc:401] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 election: Vote denied by peer bc63d858698e41f2ab32791255236e90 (127.3.230.188:43675). Message: Invalid argument: T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate eea9f8bb6b784102a362e3ab27e00de9 in current term 2: Already voted for candidate bc63d858698e41f2ab32791255236e90 in this term. 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617871 4035 leader_election.cc:278] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617907 4690 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 2.083s (election complete) 04:43:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:53.617914 4690 raft_consensus.cc:2497] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority 04:43:53.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.661672 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:53.662647 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:54.112788 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:54.662173 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:54.663031 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:54.828 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:54.828 [DEBUG - New I/O worker #7] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:54.829 [INFO - New I/O worker #7] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:54.829 [WARN - New I/O worker #7] (ConnectToCluster.java:281) Unable to find the leader master 127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922; will retry 04:43:54.829 [DEBUG - New I/O worker #7] (Deferred.java:1330) callback=retry RPC after error@1050661487 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416) -> (continuation of Deferred@2073910232 after retry RPC after error@1050661487), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416) -> (continuation of Deferred@2073910232 after retry RPC after error@1050661487)), so the following Deferred is getting paused: Deferred@2073910232(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.113281 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134068 4697 raft_consensus.cc:457] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134099 4697 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 1.757s (starting election) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134133 4697 raft_consensus.cc:479] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134189 4697 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 pre-election: Requesting pre-vote from peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134229 4697 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 pre-election: Requesting pre-vote from peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.134407 4161 leader_election.cc:307] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134660 4063 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bc63d858698e41f2ab32791255236e90" candidate_term: 3 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "eea9f8bb6b784102a362e3ab27e00de9" is_pre_election: true 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134688 4063 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 2.945s (vote granted) 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134701 4063 raft_consensus.cc:2255] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bc63d858698e41f2ab32791255236e90 in term 2. 04:43:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134783 4160 leader_election.cc:387] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 pre-election: Vote granted by peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134790 4160 leader_election.cc:278] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134843 4697 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 3.203s (election complete) 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.134928 4697 raft_consensus.cc:2549] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Leader pre-election won for term 3 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.135013 4697 raft_consensus.cc:457] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.135026 4697 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 2.342s (starting election) 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.135033 4697 raft_consensus.cc:2794] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 2 FOLLOWER]: Advancing to term 3 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.135800 4697 raft_consensus.cc:479] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.135850 4697 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 election: Requesting vote from peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.135866 4697 leader_election.cc:251] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 election: Requesting vote from peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922) 04:43:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.136024 4161 leader_election.cc:307] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136231 4063 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bc63d858698e41f2ab32791255236e90" candidate_term: 3 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "eea9f8bb6b784102a362e3ab27e00de9" 04:43:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136247 4063 raft_consensus.cc:2794] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 2 FOLLOWER]: Advancing to term 3 04:43:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136258 4063 raft_consensus.cc:2742] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9: Snoozing failure detection for 2.037s (vote granted) 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136781 4063 raft_consensus.cc:2255] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bc63d858698e41f2ab32791255236e90 in term 3. 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136864 4160 leader_election.cc:387] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 election: Vote granted by peer eea9f8bb6b784102a362e3ab27e00de9 (127.3.230.190:46041) 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136873 4160 leader_election.cc:278] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136917 4697 raft_consensus.cc:2742] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Snoozing failure detection for 3.333s (election complete) 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136930 4697 raft_consensus.cc:2549] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 3 FOLLOWER]: Leader election won for term 3 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.136974 4697 raft_consensus.cc:595] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [term 3 LEADER]: Becoming Leader. State: Replica: bc63d858698e41f2ab32791255236e90, State: Running, Role: LEADER 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.137076 4697 consensus_queue.cc:205] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.137396 4697 sys_catalog.cc:337] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bc63d858698e41f2ab32791255236e90. Latest consensus state: current_term: 3 leader_uuid: "bc63d858698e41f2ab32791255236e90" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } } 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.137408 4697 sys_catalog.cc:340] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [sys.catalog]: This master's current role is: LEADER 04:43:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.137753 4701 catalog_manager.cc:1026] Loading table and tablet metadata into memory... 04:43:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.137992 4701 catalog_manager.cc:343] Loaded metadata for table TestMasterFailover-killBefore=OPEN_TABLE [id=c5cc0f2dcd0b4747a1f944fd14bcdbf9] 04:43:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.138222 4701 catalog_manager.cc:397] Loaded metadata for tablet dc16229ae54d418c82f07b64e4faf2ea (table TestMasterFailover-killBefore=OPEN_TABLE [id=c5cc0f2dcd0b4747a1f944fd14bcdbf9]) 04:43:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.138358 4701 catalog_manager.cc:1037] Initializing Kudu internal certificate authority... 04:43:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.138659 4701 catalog_manager.cc:1046] Loading token signing keys... 04:43:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.138833 4701 catalog_manager.cc:4073] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90: Loaded TSK: 0 04:43:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.139307 4063 raft_consensus.cc:1083] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [term 3 FOLLOWER]: Refusing update from remote peer bc63d858698e41f2ab32791255236e90: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 3 index: 9. (index mismatch) 04:43:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.139310 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 1 times. 04:43:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.139433 4699 consensus_queue.cc:963] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s 04:43:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.139823 4703 sys_catalog.cc:337] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bc63d858698e41f2ab32791255236e90. Latest consensus state: current_term: 3 leader_uuid: "bc63d858698e41f2ab32791255236e90" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } } 04:43:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.139837 4703 sys_catalog.cc:340] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [sys.catalog]: This master's current role is: FOLLOWER 04:43:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.140086 4697 sys_catalog.cc:337] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "bc63d858698e41f2ab32791255236e90" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } } 04:43:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.140100 4697 sys_catalog.cc:340] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 [sys.catalog]: This master's current role is: LEADER 04:43:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.140687 4703 sys_catalog.cc:337] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "bc63d858698e41f2ab32791255236e90" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eea9f8bb6b784102a362e3ab27e00de9" member_type: VOTER last_known_addr { host: "127.3.230.190" port: 46041 } } peers { permanent_uuid: "87c8672d5f194f769df808882366f81b" member_type: VOTER last_known_addr { host: "127.3.230.189" port: 53922 } } peers { permanent_uuid: "bc63d858698e41f2ab32791255236e90" member_type: VOTER last_known_addr { host: "127.3.230.188" port: 43675 } } } 04:43:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:55.140823 4703 sys_catalog.cc:340] T 00000000000000000000000000000000 P eea9f8bb6b784102a362e3ab27e00de9 [sys.catalog]: This master's current role is: FOLLOWER 04:43:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.628376 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 2 times. 04:43:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.662621 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:55.663475 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:56.030972 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 3 times. 04:43:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:56.113772 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:56.441660 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 4 times. 04:43:56.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:56.663136 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:56.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:56.663506 4342 heartbeater.cc:467] Master 127.3.230.188:43675 was elected leader, sending a full tablet report... 04:43:56.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:56.663884 4485 heartbeater.cc:467] Master 127.3.230.188:43675 was elected leader, sending a full tablet report... 04:43:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:56.663930 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:57.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:57.002774 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 5 times. 04:43:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:57.114296 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:57.114557 4611 heartbeater.cc:467] Master 127.3.230.188:43675 was elected leader, sending a full tablet report... 04:43:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:57.620097 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 6 times. 04:43:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:57.663604 4341 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:57.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:57.664335 4484 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.098987 4161 consensus_peers.cc:433] T 00000000000000000000000000000000 P bc63d858698e41f2ab32791255236e90 -> Peer 87c8672d5f194f769df808882366f81b (127.3.230.189:53922): Couldn't send request to peer 87c8672d5f194f769df808882366f81b for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 7 times. 04:43:58.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.114750 4610 heartbeater.cc:587] Failed to heartbeat to 127.3.230.189:53922: Network error: Failed to ping master at 127.3.230.189:53922: Client connection negotiation failed: client connection to 127.3.230.189:53922: connect: Connection refused (error 111) 04:43:58.226 [INFO - New I/O boss #9] (Connection.java:439) Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922 04:43:58.226 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1985) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 04:43:58.228 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=Kudu Master, attempt=14, DeadlineTracker(timeout=50000, elapsed=6097), Traces: [0ms] sending RPC to server master-127.3.230.189:53922, [1ms] received from server master-127.3.230.189:53922 response Network error: connection disconnected, [1ms] delaying RPC due to Network error: connection disconnected, [14ms] querying master, [14ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [14ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [15ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [15ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [15ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [16ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [18ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [34ms] querying master, [34ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [34ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [35ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [35ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [35ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [35ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [36ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [54ms] querying master, [54ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [54ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [55ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [55ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [55ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [55ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [56ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [74ms] querying master, [74ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [74ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [75ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [75ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [75ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [75ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [76ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [114ms] querying master, [114ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [114ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [115ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [115ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [115ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [116ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [116ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [153ms] querying master, [153ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [153ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [154ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [154ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [154ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [154ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [155ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [233ms] querying master, [234ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [234ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [234ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [235ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [235ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [235ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [236ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [373ms] querying master, [374ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [374ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [374ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [374ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [374ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [375ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [375ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [513ms] querying master, [513ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [514ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [514ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [514ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [514ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [515ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [515ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [734ms] querying master, [734ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [734ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [734ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [735ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [735ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [735ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [736ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [774ms] querying master, [774ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [774ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [774ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [775ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [775ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [775ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [776ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.230.189:53922(127.3.230.189:53922): Connection refused: /127.3.230.189:53922, [2694ms] querying master, [2694ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.190:46041, [2694ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.188:43675, [2694ms] Sub rpc: ConnectToMaster sending RPC to server master-127.3.230.189:53922, [2695ms] Sub rpc: ConnectToMaster received from server master-127.3.230.188:43675 response OK, [2695ms] Sub rpc: ConnectToMaster received from server master-127.3.230.190:46041 response OK, [2699ms] Sub rpc: ConnectToMaster received from server master-127.3.230.189:53922 response Network error: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [2700ms] delaying RPC due to Service unavailable: Master config (127.3.230.190:46041,127.3.230.188:43675,127.3.230.189:53922) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.3.230.189:53922, [6093ms] querying master, [6093ms] trace too long, truncated, Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416) -> (continuation of Deferred@2073910232 after retry RPC after error@1050661487), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416) -> (continuation of Deferred@2073910232 after retry RPC after error@1050661487))) after lookup 04:43:58.228 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=retry RPC@738769097 returned Deferred@2138813040(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2821d152 -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416) -> (continuation of Deferred@2073910232 after retry RPC after error@1050661487) -> (continuation of Deferred@965223135 after retry RPC@738769097), errback=passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354430464 after retry RPC after error@704061551) -> (continuation of Deferred@66121171 after retry RPC after error@221712901) -> (continuation of Deferred@1075782980 after retry RPC after error@550998666) -> (continuation of Deferred@983585251 after retry RPC after error@879212831) -> (continuation of Deferred@1461087238 after retry RPC after error@1506904396) -> (continuation of Deferred@2069345643 after retry RPC after error@707264731) -> (continuation of Deferred@385987481 after retry RPC after error@1311970744) -> (continuation of Deferred@242333022 after retry RPC after error@1849405696) -> (continuation of Deferred@1645517945 after retry RPC after error@1088314912) -> (continuation of Deferred@235923358 after retry RPC after error@2071608599) -> (continuation of Deferred@853943481 after retry RPC after error@417210416) -> (continuation of Deferred@2073910232 after retry RPC after error@1050661487) -> (continuation of Deferred@965223135 after retry RPC@738769097)), so the following Deferred is getting paused: Deferred@965223135(state=PAUSED, result=Deferred@2138813040, callback=, errback=) 04:43:58.226 [DEBUG - New I/O worker #8] (Connection.java:688) [peer master-127.3.230.189:53922(127.3.230.189:53922)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:58.230 [INFO - New I/O worker #8] (ConnectToCluster.java:361) Unable to connect to master 127.3.230.189:53922: java.net.ConnectException: Connection refused: /127.3.230.189:53922 04:43:58.230 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:737) Opened table c5cc0f2dcd0b4747a1f944fd14bcdbf9 04:43:58.244 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1985) Learned about tablet dc16229ae54d418c82f07b64e4faf2ea for table 'TestMasterFailover-killBefore=OPEN_TABLE' with partition [, ) 04:43:58.251 [DEBUG - New I/O worker #2] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=dc16229ae54d418c82f07b64e4faf2ea@[0257d621b4a5457098a36cfdbb8f329d(127.3.230.131:47182)[L],5f67e058ce8e485998eb40d421b840f1(127.3.230.130:55584),6e525d66c3444766a62305b33618b74d(127.3.230.129:51896)]}] 04:43:58.253 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1150) Retrying sending RPC ScanRequest(scannerId=null, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=13), Traces: [0ms] querying master, [1ms] Sub rpc: GetTableLocations sending RPC to server master-127.3.230.188:43675, [4ms] Sub rpc: GetTableLocations received from server master-127.3.230.188:43675 response OK, null)) after lookup 04:43:58.261 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=retry RPC@460220140 returned Deferred@1740822490(state=PENDING, result=null, callback=(continuation of Deferred@1828268569 after retry RPC@460220140), errback=(continuation of Deferred@1828268569 after retry RPC@460220140)), so the following Deferred is getting paused: Deferred@1828268569(state=PAUSED, result=Deferred@1740822490, callback=scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@42185175 -> wakeup thread Time-limited test, errback=passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test) 04:43:58.300 [DEBUG - New I/O worker #1] (Negotiator.java:764) Authenticated connection [id: 0x82816f81, /127.3.230.131:33075 => /127.3.230.131:47182] using TOKEN/null 04:43:58.321 [DEBUG - New I/O worker #1] (AsyncKuduScanner.java:1030) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 0 rows, more = false) for scanner KuduScanner(table=TestMasterFailover-killBefore=OPEN_TABLE, tablet=dc16229ae54d418c82f07b64e4faf2ea, scannerId=null, scanRequestTimeout=30000) 04:43:58.324 [DEBUG - Time-limited test] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:43:58.326 [DEBUG - New I/O worker #1] (Connection.java:688) [peer master-127.3.230.190:46041(127.3.230.190:46041)] cleaning up while in state READY due to: connection disconnected 04:43:58.327 [DEBUG - New I/O worker #2] (Connection.java:688) [peer master-127.3.230.188:43675(127.3.230.188:43675)] cleaning up while in state READY due to: connection disconnected 04:43:58.328 [DEBUG - New I/O worker #1] (Connection.java:688) [peer 0257d621b4a5457098a36cfdbb8f329d(127.3.230.131:47182)] cleaning up while in state READY due to: connection disconnected 04:43:58.328 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:43:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.330904 3994 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 4214 04:43:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.333892 4035 connection.cc:512] server connection from 127.3.230.129:40623 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.129:40623): Connection reset by peer (error 104) 04:43:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.333899 4160 connection.cc:512] server connection from 127.3.230.129:38724 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.129:38724): Connection reset by peer (error 104) 04:43:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.333925 3994 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 4347 04:43:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.334051 4498 connection.cc:512] client connection to 127.3.230.129:51896 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.129:51896): Connection reset by peer (error 104) 04:43:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.336493 4158 connection.cc:512] server connection from 127.3.230.130:45246 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.130:45246): Connection reset by peer (error 104) 04:43:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.336499 4034 connection.cc:512] server connection from 127.3.230.130:39386 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.130:39386): Connection reset by peer (error 104) 04:43:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.336515 4498 connection.cc:512] client connection to 127.3.230.130:55584 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.130:55584): Connection reset by peer (error 104) 04:43:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.336531 3994 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 4490 04:43:58.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.338645 4160 connection.cc:512] server connection from 127.3.230.131:56466 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.131:56466): Connection reset by peer (error 104) 04:43:58.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.338646 4037 connection.cc:512] server connection from 127.3.230.131:46582 recv error: Network error: failed to read from TLS socket (remote: 127.3.230.131:46582): Connection reset by peer (error 104) 04:43:58.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.338673 3994 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 4002 04:43:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.340952 3994 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 4151 04:43:58.348 [INFO - Test worker] (RandomUtils.java:46) Using random seed: 1538221438348 04:43:58.349 [INFO - Test worker] (BaseKuduTest.java:77) Creating a new MiniKuduCluster... 04:43:58.349 [INFO - Test worker] (MiniKuduCluster.java:144) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 04:43:58.349 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 04:43:58.349 [INFO - Test worker] (MiniKuduCluster.java:195) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 04:43:58.351 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621" hms_mode: NONE mini_kdc_options { } } 04:43:58.362 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:43:58.362 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: start_cluster { } 04:43:58.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.366822 4730 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/wal 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.190:55468 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.190 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/logs 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data/info.pb 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 with env {} 04:43:58.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.377439 4737 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.377539 4737 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.378484 4737 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:43:58.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.378501 4737 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.378507 4737 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:43:58.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.378512 4737 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:43:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.379714 4737 master_main.cc:73] Master server non-default flags: 04:43:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data 04:43:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/wal 04:43:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.190:55468 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data/info.pb 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.190 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.4737 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/logs 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.382812 4737 master_main.cc:80] Initializing master server... 04:43:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.383018 4737 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.383114 4737 fs_manager.cc:262] Metadata directory not provided 04:43:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.383131 4737 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.383306 4737 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.383313 4737 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.384110 4737 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data/instance: 04:43:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" 04:43:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.384496 4737 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/wal/instance: 04:43:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" 04:43:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.387223 4737 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.001s 04:43:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.387454 4737 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.387518 4737 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.388500 4737 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.388528 4737 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/wal 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.388573 4737 fs_report.cc:352] FS layout report 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/wal 04:43:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/wal 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data/data 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.417384 4737 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.417558 4737 master_main.cc:83] Starting Master server... 04:43:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.420722 4737 rpc_server.cc:205] RPC server started. Bound to: 127.4.158.190:55468 04:43:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.420769 4737 webserver.cc:175] Starting webserver on 127.4.158.190:0 04:43:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.420773 4737 webserver.cc:186] Document root disabled 04:43:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.420894 4737 webserver.cc:313] Webserver started. Bound to: http://127.4.158.190:57733/ 04:43:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.421082 4737 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-0/data/info.pb 04:43:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.421335 4799 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.422694 4799 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.425077 4799 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.425320 4799 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.4.158.189:44906: Network error: Client connection negotiation failed: client connection to 127.4.158.189:44906: connect: Connection refused (error 111) 04:43:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.429535 4730 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 4737 04:43:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.429778 4730 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/wal 04:43:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.189:44906 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.189 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/logs 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data/info.pb 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:43:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 with env {} 04:43:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.439942 4802 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.440033 4802 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.440837 4802 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:43:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.440852 4802 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.440855 4802 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:43:58.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.440860 4802 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:43:58.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.441882 4802 master_main.cc:73] Master server non-default flags: 04:43:58.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/wal 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.189:44906 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data/info.pb 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.189 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.4802 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/logs 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.443495 4802 master_main.cc:80] Initializing master server... 04:43:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.443646 4802 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.443742 4802 fs_manager.cc:262] Metadata directory not provided 04:43:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.443766 4802 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.443820 4802 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.443827 4802 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.444500 4802 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data/instance: 04:43:58.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "ff6b866d57eb434baa9075f71fec0bab" 04:43:58.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.444502 4799 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } attempt: 1 04:43:58.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.444741 4799 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.4.158.189:44906: Network error: Client connection negotiation failed: client connection to 127.4.158.189:44906: connect: Connection refused (error 111) 04:43:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.445024 4802 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/wal/instance: 04:43:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "ff6b866d57eb434baa9075f71fec0bab" 04:43:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.447013 4802 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:43:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.447180 4802 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.447355 4802 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.448266 4802 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.448292 4802 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/wal 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "ff6b866d57eb434baa9075f71fec0bab" 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.448336 4802 fs_report.cc:352] FS layout report 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/wal 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/wal 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data/data 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.463987 4802 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.464201 4802 master_main.cc:83] Starting Master server... 04:43:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.467177 4802 rpc_server.cc:205] RPC server started. Bound to: 127.4.158.189:44906 04:43:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.467341 4802 webserver.cc:175] Starting webserver on 127.4.158.189:0 04:43:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.467355 4802 webserver.cc:186] Document root disabled 04:43:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.467510 4802 webserver.cc:313] Webserver started. Bound to: http://127.4.158.189:58920/ 04:43:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.467736 4802 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-1/data/info.pb 04:43:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.467967 4862 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.470003 4862 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.471267 4730 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 4802 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.471581 4730 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/wal 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.188:46940 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.188 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/logs 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data/info.pb 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 with env {} 04:43:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.473497 4862 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.476073 4862 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.476382 4862 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.4.158.188:46940: Network error: Client connection negotiation failed: client connection to 127.4.158.188:46940: connect: Connection refused (error 111) 04:43:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.484112 4864 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.484230 4864 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.485235 4864 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:43:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.485252 4864 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.485258 4864 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:43:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.485263 4864 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.486491 4864 master_main.cc:73] Master server non-default flags: 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/wal 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.188:46940 04:43:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data/info.pb 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.188 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.4864 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/logs 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.489707 4864 master_main.cc:80] Initializing master server... 04:43:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.489854 4864 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.489943 4864 fs_manager.cc:262] Metadata directory not provided 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.489959 4864 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.490012 4864 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.490018 4864 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.490715 4864 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data/instance: 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a31861fbd1ac469d97c7263996794272" 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.490945 4864 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/wal/instance: 04:43:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a31861fbd1ac469d97c7263996794272" 04:43:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.492908 4799 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } attempt: 2 04:43:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.492987 4864 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.001s 04:43:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.493171 4864 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.493373 4864 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.494531 4864 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.494709 4864 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/wal 04:43:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a31861fbd1ac469d97c7263996794272" 04:43:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.494976 4864 fs_report.cc:352] FS layout report 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/wal 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/wal 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data/data 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.495205 4799 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.495383 4799 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.4.158.188:46940: Network error: Client connection negotiation failed: client connection to 127.4.158.188:46940: connect: Connection refused (error 111) 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.495574 4862 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } attempt: 1 04:43:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.495724 4862 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.4.158.188:46940: Network error: Client connection negotiation failed: client connection to 127.4.158.188:46940: connect: Connection refused (error 111) 04:43:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.521312 4864 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.521466 4864 master_main.cc:83] Starting Master server... 04:43:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.524235 4864 rpc_server.cc:205] RPC server started. Bound to: 127.4.158.188:46940 04:43:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.524294 4864 webserver.cc:175] Starting webserver on 127.4.158.188:0 04:43:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.524302 4864 webserver.cc:186] Document root disabled 04:43:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.524454 4864 webserver.cc:313] Webserver started. Bound to: http://127.4.158.188:34993/ 04:43:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.524580 4864 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/master-2/data/info.pb 04:43:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.524910 4924 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.525275 4730 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 4864 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.525651 4730 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/wal 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.129:0 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.4.158.129 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.129 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/logs 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data/info.pb 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:43:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.527854 4924 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.530015 4924 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.534512 4924 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.536790 4924 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: Bootstrap starting. 04:43:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.537044 4924 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: No blocks or log segments found. Creating new log. 04:43:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.537217 4924 log.cc:525] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: Log is configured to *not* fsync() on all Append() calls 04:43:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.537797 4924 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: No bootstrap required, opened a new log 04:43:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.538609 4925 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.538730 4925 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539055 4924 raft_consensus.cc:325] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539075 4924 raft_consensus.cc:351] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539085 4924 raft_consensus.cc:630] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a31861fbd1ac469d97c7263996794272, State: Initialized, Role: FOLLOWER 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539146 4924 consensus_queue.cc:228] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539253 4924 sys_catalog.cc:337] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539265 4924 sys_catalog.cc:340] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539324 4924 sys_catalog.cc:424] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: configured and running, proceeding with master startup. 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539428 4929 sys_catalog.cc:337] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539439 4929 sys_catalog.cc:340] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.539573 4864 master_main.cc:86] Master server successfully started. 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.539731 4925 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.539880 4925 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.158.129 04:43:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.540238 4930 catalog_manager.cc:1084] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:43:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.540356 4930 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.541260 4925 tablet_server_main.cc:78] Tablet server non-default flags: 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/wal 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.129:0 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data/info.pb 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.129 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.4925 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.4.158.129 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/logs 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.543328 4925 tablet_server_main.cc:85] Initializing tablet server... 04:43:58.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.543910 4862 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } attempt: 2 04:43:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.544330 4925 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.544581 4925 fs_manager.cc:262] Metadata directory not provided 04:43:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.544770 4925 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.544997 4925 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.545197 4925 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.546293 4925 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data/instance: 04:43:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "dfc0dc73364b4de5a5dfbe9f586ae9bd" 04:43:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.546497 4862 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: Bootstrap starting. 04:43:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.546747 4925 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/wal/instance: 04:43:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "dfc0dc73364b4de5a5dfbe9f586ae9bd" 04:43:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.546854 4862 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: No blocks or log segments found. Creating new log. 04:43:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.547000 4862 log.cc:525] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: Log is configured to *not* fsync() on all Append() calls 04:43:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.549527 4925 fs_manager.cc:502] Time spent creating directory manager: real 0.003s user 0.000s sys 0.002s 04:43:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.549700 4925 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.549762 4925 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.551051 4862 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: No bootstrap required, opened a new log 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552475 4862 raft_consensus.cc:325] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552496 4862 raft_consensus.cc:351] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552505 4862 raft_consensus.cc:630] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff6b866d57eb434baa9075f71fec0bab, State: Initialized, Role: FOLLOWER 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552562 4862 consensus_queue.cc:228] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552642 4862 sys_catalog.cc:337] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552652 4862 sys_catalog.cc:340] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552700 4862 sys_catalog.cc:424] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: configured and running, proceeding with master startup. 04:43:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552800 4925 fs_manager.cc:418] Time spent opening block manager: real 0.002s user 0.000s sys 0.000s 04:43:58.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552876 4802 master_main.cc:86] Master server successfully started. 04:43:58.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.553369 4939 sys_catalog.cc:337] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.553567 4939 sys_catalog.cc:340] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.552820 4925 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/wal 04:43:58.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "dfc0dc73364b4de5a5dfbe9f586ae9bd" 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.553887 4925 fs_report.cc:352] FS layout report 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/wal 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/wal 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data/data 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.554193 4940 catalog_manager.cc:1084] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:43:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.554204 4940 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:43:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.558558 4799 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } attempt: 1 04:43:58.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.561143 4799 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Bootstrap starting. 04:43:58.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.561437 4799 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: No blocks or log segments found. Creating new log. 04:43:58.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.561590 4799 log.cc:525] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Log is configured to *not* fsync() on all Append() calls 04:43:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.562296 4799 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: No bootstrap required, opened a new log 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563635 4799 raft_consensus.cc:325] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563649 4799 raft_consensus.cc:351] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563657 4799 raft_consensus.cc:630] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7f7919531cc47fa995bbbfc1dd2cca6, State: Initialized, Role: FOLLOWER 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563722 4799 consensus_queue.cc:228] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563802 4799 sys_catalog.cc:337] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563812 4799 sys_catalog.cc:340] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.563860 4799 sys_catalog.cc:424] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: configured and running, proceeding with master startup. 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.564712 4737 master_main.cc:86] Master server successfully started. 04:43:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.564715 4947 sys_catalog.cc:337] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.565088 4947 sys_catalog.cc:340] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.567528 4948 catalog_manager.cc:1084] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:43:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.567545 4948 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:43:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.574538 4925 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.574759 4925 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:43:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.574808 4925 tablet_server_main.cc:90] Starting tablet server... 04:43:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.577787 4925 rpc_server.cc:205] RPC server started. Bound to: 127.4.158.129:48500 04:43:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.577850 4925 webserver.cc:175] Starting webserver on 127.4.158.129:0 04:43:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.577857 4925 webserver.cc:186] Document root disabled 04:43:58.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.578022 4925 webserver.cc:313] Webserver started. Bound to: http://127.4.158.129:58126/ 04:43:58.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.578136 4925 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-0/data/info.pb 04:43:58.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.582355 4925 tablet_server_main.cc:93] Tablet server successfully started. 04:43:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.582974 5060 heartbeater.cc:346] Connected to a master server at 127.4.158.188:46940 04:43:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.582989 5060 heartbeater.cc:426] Registering TS with master... 04:43:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.583029 5060 heartbeater.cc:475] Master 127.4.158.188:46940 requested a full tablet report, sending... 04:43:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.583631 4891 ts_manager.cc:98] Registered new tserver with Master: dfc0dc73364b4de5a5dfbe9f586ae9bd (127.4.158.129:48500) 04:43:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.586261 5059 heartbeater.cc:346] Connected to a master server at 127.4.158.189:44906 04:43:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.586277 5059 heartbeater.cc:426] Registering TS with master... 04:43:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.586320 5059 heartbeater.cc:475] Master 127.4.158.189:44906 requested a full tablet report, sending... 04:43:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.586827 4828 ts_manager.cc:98] Registered new tserver with Master: dfc0dc73364b4de5a5dfbe9f586ae9bd (127.4.158.129:48500) 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.587692 4730 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 4925 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.587966 4730 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/wal 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.130:0 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.4.158.130 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.130 04:43:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/logs 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data/info.pb 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.588893 5058 heartbeater.cc:346] Connected to a master server at 127.4.158.190:55468 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.588907 5058 heartbeater.cc:426] Registering TS with master... 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.588940 5058 heartbeater.cc:475] Master 127.4.158.190:55468 requested a full tablet report, sending... 04:43:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.589265 4766 ts_manager.cc:98] Registered new tserver with Master: dfc0dc73364b4de5a5dfbe9f586ae9bd (127.4.158.129:48500) 04:43:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.605355 5064 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.605872 5064 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.607040 5064 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.607264 5064 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.158.130 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.608636 5064 tablet_server_main.cc:78] Tablet server non-default flags: 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/wal 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.130:0 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data/info.pb 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.130 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.5064 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.4.158.130 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/logs 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.612505 5064 tablet_server_main.cc:85] Initializing tablet server... 04:43:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.612946 5064 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.613210 5064 fs_manager.cc:262] Metadata directory not provided 04:43:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.613428 5064 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.613687 5064 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.613833 5064 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.614789 5064 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data/instance: 04:43:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "b22cf4a570454e80884c8247124a88cc" 04:43:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.615308 5064 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/wal/instance: 04:43:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "b22cf4a570454e80884c8247124a88cc" 04:43:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.620234 5064 fs_manager.cc:502] Time spent creating directory manager: real 0.004s user 0.000s sys 0.002s 04:43:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.620553 5064 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.620751 5064 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.622076 5064 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.002s 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.622222 5064 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/wal 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "b22cf4a570454e80884c8247124a88cc" 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.622470 5064 fs_report.cc:352] FS layout report 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/wal 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/wal 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data/data 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.640470 5064 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.640867 5064 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:43:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.641027 5064 tablet_server_main.cc:90] Starting tablet server... 04:43:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.644345 5064 rpc_server.cc:205] RPC server started. Bound to: 127.4.158.130:55016 04:43:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.644511 5064 webserver.cc:175] Starting webserver on 127.4.158.130:0 04:43:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.644619 5064 webserver.cc:186] Document root disabled 04:43:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.644886 5064 webserver.cc:313] Webserver started. Bound to: http://127.4.158.130:56079/ 04:43:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.645117 5064 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-1/data/info.pb 04:43:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.645521 5064 tablet_server_main.cc:93] Tablet server successfully started. 04:43:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.650125 5184 heartbeater.cc:346] Connected to a master server at 127.4.158.189:44906 04:43:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.650141 5184 heartbeater.cc:426] Registering TS with master... 04:43:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.650204 5184 heartbeater.cc:475] Master 127.4.158.189:44906 requested a full tablet report, sending... 04:43:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.650781 4828 ts_manager.cc:98] Registered new tserver with Master: b22cf4a570454e80884c8247124a88cc (127.4.158.130:55016) 04:43:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.653146 5185 heartbeater.cc:346] Connected to a master server at 127.4.158.188:46940 04:43:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.653167 5185 heartbeater.cc:426] Registering TS with master... 04:43:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.653203 5185 heartbeater.cc:475] Master 127.4.158.188:46940 requested a full tablet report, sending... 04:43:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.653518 4891 ts_manager.cc:98] Registered new tserver with Master: b22cf4a570454e80884c8247124a88cc (127.4.158.130:55016) 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.654136 4730 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 5064 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.654453 4730 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/wal 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.131:0 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.4.158.131 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.131 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/logs 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data/info.pb 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:43:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.656309 5183 heartbeater.cc:346] Connected to a master server at 127.4.158.190:55468 04:43:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.656325 5183 heartbeater.cc:426] Registering TS with master... 04:43:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.656358 5183 heartbeater.cc:475] Master 127.4.158.190:55468 requested a full tablet report, sending... 04:43:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.656621 4766 ts_manager.cc:98] Registered new tserver with Master: b22cf4a570454e80884c8247124a88cc (127.4.158.130:55016) 04:43:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.667826 4755 tablet.cc:1728] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.674474 5190 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.674958 5190 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.676076 5190 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.676276 5190 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.158.131 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.677624 5190 tablet_server_main.cc:78] Tablet server non-default flags: 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/wal 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.4.158.131:0 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data/info.pb 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.4.158.131 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.4.158.190:55468,127.4.158.189:44906,127.4.158.188:46940 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.5190 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.4.158.131 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/logs 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.681391 5190 tablet_server_main.cc:85] Initializing tablet server... 04:43:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.681815 5190 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.682054 5190 fs_manager.cc:262] Metadata directory not provided 04:43:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.682318 5190 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.682529 5190 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.682683 5190 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.683670 5190 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data/instance: 04:43:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "33d8b35df3574f3898e72326d7a552fc" 04:43:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.684226 5190 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/wal/instance: 04:43:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "33d8b35df3574f3898e72326d7a552fc" 04:43:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.689211 5190 fs_manager.cc:502] Time spent creating directory manager: real 0.004s user 0.000s sys 0.002s 04:43:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.689381 5190 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.689467 5190 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.690985 5190 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.691009 5190 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/wal 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "33d8b35df3574f3898e72326d7a552fc" 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.691052 5190 fs_report.cc:352] FS layout report 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/wal 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/wal 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data/data 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.714589 4817 tablet.cc:1728] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.720880 5190 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.721252 5190 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:43:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.721336 5190 tablet_server_main.cc:90] Starting tablet server... 04:43:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.727233 5190 rpc_server.cc:205] RPC server started. Bound to: 127.4.158.131:47880 04:43:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.727382 5190 webserver.cc:175] Starting webserver on 127.4.158.131:0 04:43:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.727475 5190 webserver.cc:186] Document root disabled 04:43:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.727730 5190 webserver.cc:313] Webserver started. Bound to: http://127.4.158.131:48221/ 04:43:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.727871 5190 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6107190293763789621/ts-2/data/info.pb 04:43:58.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.728637 5190 tablet_server_main.cc:93] Tablet server successfully started. 04:43:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.731890 5310 heartbeater.cc:346] Connected to a master server at 127.4.158.189:44906 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732164 5310 heartbeater.cc:426] Registering TS with master... 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732194 5311 heartbeater.cc:346] Connected to a master server at 127.4.158.188:46940 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732282 5309 heartbeater.cc:346] Connected to a master server at 127.4.158.190:55468 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732393 5311 heartbeater.cc:426] Registering TS with master... 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732403 5309 heartbeater.cc:426] Registering TS with master... 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732403 5310 heartbeater.cc:475] Master 127.4.158.189:44906 requested a full tablet report, sending... 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732435 5311 heartbeater.cc:475] Master 127.4.158.188:46940 requested a full tablet report, sending... 04:43:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732441 5309 heartbeater.cc:475] Master 127.4.158.190:55468 requested a full tablet report, sending... 04:43:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.732939 4828 ts_manager.cc:98] Registered new tserver with Master: 33d8b35df3574f3898e72326d7a552fc (127.4.158.131:47880) 04:43:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.733194 4766 ts_manager.cc:98] Registered new tserver with Master: 33d8b35df3574f3898e72326d7a552fc (127.4.158.131:47880) 04:43:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.733326 4891 ts_manager.cc:98] Registered new tserver with Master: 33d8b35df3574f3898e72326d7a552fc (127.4.158.131:47880) 04:43:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.736040 4730 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 5190 04:43:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.742208 4730 external_mini_cluster.cc:513] 3 TS(s) registered with all masters 04:43:58.742 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:43:58.743 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_masters { } 04:43:58.744 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.4.158.190" port: 55468 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.4.158.189" port: 44906 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.4.158.188" port: 46940 } } } 04:43:58.744 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_tservers { } 04:43:58.748 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.4.158.129" port: 48500 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.4.158.130" port: 55016 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.4.158.131" port: 47880 } } } 04:43:58.748 [INFO - Test worker] (BaseKuduTest.java:81) Creating a new Kudu client... 04:43:58.753 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:43:58.761 [DEBUG - New I/O worker #19] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.4.158.190 04:43:58.761 [DEBUG - New I/O worker #20] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.4.158.188 04:43:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764472 4947 raft_consensus.cc:457] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:43:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764503 4947 raft_consensus.cc:2742] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Snoozing failure detection for 1.616s (starting election) 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764557 4947 raft_consensus.cc:479] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764621 4947 leader_election.cc:251] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer ff6b866d57eb434baa9075f71fec0bab (127.4.158.189:44906) 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764657 4947 leader_election.cc:251] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer a31861fbd1ac469d97c7263996794272 (127.4.158.188:46940) 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764816 4901 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a31861fbd1ac469d97c7263996794272" is_pre_election: true 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764853 4901 raft_consensus.cc:2742] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: Snoozing failure detection for 1.623s (vote granted) 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764868 4901 raft_consensus.cc:2255] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7f7919531cc47fa995bbbfc1dd2cca6 in term 0. 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764914 4839 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff6b866d57eb434baa9075f71fec0bab" is_pre_election: true 04:43:58.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764938 4839 raft_consensus.cc:2742] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: Snoozing failure detection for 2.062s (vote granted) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764947 4839 raft_consensus.cc:2255] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7f7919531cc47fa995bbbfc1dd2cca6 in term 0. 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.764991 4750 leader_election.cc:387] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 pre-election: Vote granted by peer a31861fbd1ac469d97c7263996794272 (127.4.158.188:46940) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765003 4750 leader_election.cc:278] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765040 4750 leader_election.cc:387] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 pre-election: Vote granted by peer ff6b866d57eb434baa9075f71fec0bab (127.4.158.189:44906) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765045 4947 raft_consensus.cc:2742] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Snoozing failure detection for 1.883s (election complete) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765054 4947 raft_consensus.cc:2549] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Leader pre-election won for term 1 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765058 4947 raft_consensus.cc:457] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765061 4947 raft_consensus.cc:2742] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Snoozing failure detection for 2.141s (starting election) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765067 4947 raft_consensus.cc:2794] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 0 FOLLOWER]: Advancing to term 1 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765555 4947 raft_consensus.cc:479] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765602 4947 leader_election.cc:251] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 election: Requesting vote from peer ff6b866d57eb434baa9075f71fec0bab (127.4.158.189:44906) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765621 4947 leader_election.cc:251] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 election: Requesting vote from peer a31861fbd1ac469d97c7263996794272 (127.4.158.188:46940) 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765751 4839 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff6b866d57eb434baa9075f71fec0bab" 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765750 4901 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a31861fbd1ac469d97c7263996794272" 04:43:58.768 [DEBUG - New I/O worker #21] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.4.158.189 04:43:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765763 4839 raft_consensus.cc:2794] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 0 FOLLOWER]: Advancing to term 1 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765767 4901 raft_consensus.cc:2794] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 0 FOLLOWER]: Advancing to term 1 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765770 4839 raft_consensus.cc:2742] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab: Snoozing failure detection for 1.729s (vote granted) 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.765780 4901 raft_consensus.cc:2742] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: Snoozing failure detection for 2.102s (vote granted) 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766181 4839 raft_consensus.cc:2255] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7f7919531cc47fa995bbbfc1dd2cca6 in term 1. 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766280 4750 leader_election.cc:387] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 election: Vote granted by peer ff6b866d57eb434baa9075f71fec0bab (127.4.158.189:44906) 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766288 4750 leader_election.cc:278] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766324 4947 raft_consensus.cc:2742] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Snoozing failure detection for 1.677s (election complete) 04:43:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766332 4947 raft_consensus.cc:2549] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 1 FOLLOWER]: Leader election won for term 1 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766350 4901 raft_consensus.cc:2255] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7f7919531cc47fa995bbbfc1dd2cca6 in term 1. 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766423 4750 leader_election.cc:387] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [CANDIDATE]: Term 1 election: Vote granted by peer a31861fbd1ac469d97c7263996794272 (127.4.158.188:46940) 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766423 4947 raft_consensus.cc:595] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [term 1 LEADER]: Becoming Leader. State: Replica: d7f7919531cc47fa995bbbfc1dd2cca6, State: Running, Role: LEADER 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766486 4947 consensus_queue.cc:205] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766705 4947 sys_catalog.cc:337] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d7f7919531cc47fa995bbbfc1dd2cca6. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766714 4947 sys_catalog.cc:340] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: This master's current role is: LEADER 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.766965 5330 catalog_manager.cc:1026] Loading table and tablet metadata into memory... 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.767076 5330 catalog_manager.cc:1037] Initializing Kudu internal certificate authority... 04:43:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.771709 4880 tablet.cc:1728] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.783959 4901 raft_consensus.cc:1083] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [term 1 FOLLOWER]: Refusing update from remote peer d7f7919531cc47fa995bbbfc1dd2cca6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 04:43:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.783959 4839 raft_consensus.cc:1083] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [term 1 FOLLOWER]: Refusing update from remote peer d7f7919531cc47fa995bbbfc1dd2cca6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 04:43:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.784330 4947 consensus_queue.cc:963] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:43:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.784375 5328 consensus_queue.cc:963] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:43:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.785158 4929 sys_catalog.cc:337] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d7f7919531cc47fa995bbbfc1dd2cca6. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.785174 4929 sys_catalog.cc:340] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.785254 4939 sys_catalog.cc:337] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: SysCatalogTable state changed. Reason: New leader d7f7919531cc47fa995bbbfc1dd2cca6. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.785264 4939 sys_catalog.cc:340] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.785969 4947 sys_catalog.cc:337] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.785982 4947 sys_catalog.cc:340] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: This master's current role is: LEADER 04:43:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.786052 5330 catalog_manager.cc:924] Generated new certificate authority record 04:43:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.786414 4929 sys_catalog.cc:337] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.786427 4929 sys_catalog.cc:340] T 00000000000000000000000000000000 P a31861fbd1ac469d97c7263996794272 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.786923 5330 catalog_manager.cc:1046] Loading token signing keys... 04:43:58.787 [DEBUG - New I/O worker #20] (Negotiator.java:764) Authenticated connection [id: 0x7ab4d11a, /127.4.158.188:44026 => /127.4.158.188:46940] using SASL/PLAIN 04:43:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.789371 4947 sys_catalog.cc:337] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.789383 4947 sys_catalog.cc:340] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6 [sys.catalog]: This master's current role is: LEADER 04:43:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.789607 4939 sys_catalog.cc:337] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f7919531cc47fa995bbbfc1dd2cca6" member_type: VOTER last_known_addr { host: "127.4.158.190" port: 55468 } } peers { permanent_uuid: "ff6b866d57eb434baa9075f71fec0bab" member_type: VOTER last_known_addr { host: "127.4.158.189" port: 44906 } } peers { permanent_uuid: "a31861fbd1ac469d97c7263996794272" member_type: VOTER last_known_addr { host: "127.4.158.188" port: 46940 } } } 04:43:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.789615 4939 sys_catalog.cc:340] T 00000000000000000000000000000000 P ff6b866d57eb434baa9075f71fec0bab [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.790 [DEBUG - New I/O worker #21] (Negotiator.java:764) Authenticated connection [id: 0x45d46857, /127.4.158.189:60461 => /127.4.158.189:44906] using SASL/PLAIN 04:43:58.791 [DEBUG - New I/O worker #19] (Negotiator.java:764) Authenticated connection [id: 0x133297e0, /127.4.158.190:40438 => /127.4.158.190:55468] using SASL/PLAIN 04:43:58.793 [WARN - New I/O worker #19] (ConnectToCluster.java:278) None of the provided masters 127.4.158.190:55468,127.4.158.188:46940,127.4.158.189:44906 is a leader; will retry 04:43:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.793448 5330 catalog_manager.cc:4062] T 00000000000000000000000000000000 P d7f7919531cc47fa995bbbfc1dd2cca6: Generated new TSK 0 04:43:58.795 [DEBUG - New I/O worker #19] (Connection.java:688) [peer master-127.4.158.190:55468(127.4.158.190:55468)] cleaning up while in state READY due to: connection disconnected 04:43:58.796 [DEBUG - New I/O worker #20] (Connection.java:688) [peer master-127.4.158.188:46940(127.4.158.188:46940)] cleaning up while in state READY due to: connection disconnected 04:43:58.796 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:43:58.796 [DEBUG - New I/O worker #21] (Connection.java:688) [peer master-127.4.158.189:44906(127.4.158.189:44906)] cleaning up while in state READY due to: connection disconnected 04:43:58.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.798032 4730 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 4925 04:43:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.800047 4872 connection.cc:512] server connection from 127.4.158.129:34755 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.129:34755): Connection reset by peer (error 104) 04:43:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.800056 4749 connection.cc:512] server connection from 127.4.158.129:57934 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.129:57934): Connection reset by peer (error 104) 04:43:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.800088 4730 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 5064 04:43:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.800746 4809 connection.cc:512] server connection from 127.4.158.129:39272 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.129:39272): Connection reset by peer (error 104) 04:43:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.802163 4875 connection.cc:512] server connection from 127.4.158.130:54401 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.130:54401): Connection reset by peer (error 104) 04:43:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.802203 4750 connection.cc:512] server connection from 127.4.158.130:41871 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.130:41871): Connection reset by peer (error 104) 04:43:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.802207 4730 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 5190 04:43:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.802253 4809 connection.cc:512] server connection from 127.4.158.130:53881 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.130:53881): Connection reset by peer (error 104) 04:43:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.804466 4875 connection.cc:512] server connection from 127.4.158.131:34211 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.131:34211): Connection reset by peer (error 104) 04:43:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.804482 4812 connection.cc:512] server connection from 127.4.158.131:38069 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.131:38069): Connection reset by peer (error 104) 04:43:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.804495 4748 connection.cc:512] server connection from 127.4.158.131:49020 recv error: Network error: failed to read from TLS socket (remote: 127.4.158.131:49020): Connection reset by peer (error 104) 04:43:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.804502 4730 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 4737 04:43:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.807024 4730 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 4802 04:43:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.809168 4730 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 4864 04:43:58.817 [ERROR - Test worker] (RetryRule.java:76) testKillLeaderBeforeCreateClient(org.apache.kudu.client.TestMasterFailover): failed run 1 org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.158.190:55468,127.4.158.188:46940,127.4.158.189:44906) has no leader. at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:279) at org.apache.kudu.client.ConnectToCluster.access$100(ConnectToCluster.java:47) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:323) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:312) at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:247) at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:294) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:269) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.messageReceived(Connection.java:391) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 04:43:58.820 [INFO - Test worker] (BaseKuduTest.java:77) Creating a new MiniKuduCluster... 04:43:58.820 [INFO - Test worker] (MiniKuduCluster.java:144) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 04:43:58.821 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 04:43:58.821 [INFO - Test worker] (MiniKuduCluster.java:195) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 04:43:58.822 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255" hms_mode: NONE mini_kdc_options { } } 04:43:58.833 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:43:58.833 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: start_cluster { } 04:43:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.838634 5341 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/wal 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.126:47505 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.126 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/logs 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data/info.pb 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 with env {} 04:43:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.849336 5348 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.849436 5348 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.850282 5348 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:43:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.850299 5348 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.850304 5348 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:43:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.850311 5348 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.851423 5348 master_main.cc:73] Master server non-default flags: 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/wal 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.126:47505 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data/info.pb 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.126 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.5348 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/logs 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.853466 5348 master_main.cc:80] Initializing master server... 04:43:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.853623 5348 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.853718 5348 fs_manager.cc:262] Metadata directory not provided 04:43:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.853734 5348 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.853790 5348 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.853796 5348 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.854460 5348 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data/instance: 04:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "f6f13d895c824d258aab0ab409720547" 04:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.854691 5348 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/wal/instance: 04:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "f6f13d895c824d258aab0ab409720547" 04:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.860638 5348 fs_manager.cc:502] Time spent creating directory manager: real 0.006s user 0.000s sys 0.002s 04:43:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.860826 5348 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.860890 5348 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.861768 5348 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.861795 5348 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/wal 04:43:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "f6f13d895c824d258aab0ab409720547" 04:43:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.861840 5348 fs_report.cc:352] FS layout report 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/wal 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/wal 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data/data 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.879207 5348 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.879422 5348 master_main.cc:83] Starting Master server... 04:43:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.882086 5348 rpc_server.cc:205] RPC server started. Bound to: 127.5.55.126:47505 04:43:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.882149 5348 webserver.cc:175] Starting webserver on 127.5.55.126:0 04:43:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.882158 5348 webserver.cc:186] Document root disabled 04:43:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.882468 5348 webserver.cc:313] Webserver started. Bound to: http://127.5.55.126:58235/ 04:43:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.882642 5348 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-0/data/info.pb 04:43:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.882884 5407 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:58.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.884460 5407 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.887019 5407 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.887239 5407 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.55.125:49943: Network error: Client connection negotiation failed: client connection to 127.5.55.125:49943: connect: Connection refused (error 111) 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.891204 5341 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 5348 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.891511 5341 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/wal 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.125:49943 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.125 04:43:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/logs 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data/info.pb 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 with env {} 04:43:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.901739 5410 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.901836 5410 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.902839 5410 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:43:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.902858 5410 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.902864 5410 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:43:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.902869 5410 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.904101 5410 master_main.cc:73] Master server non-default flags: 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/wal 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.125:49943 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data/info.pb 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.125 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.5410 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/logs 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.906414 5407 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } attempt: 1 04:43:58.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.906633 5407 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.55.125:49943: Network error: Client connection negotiation failed: client connection to 127.5.55.125:49943: connect: Connection refused (error 111) 04:43:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.907542 5410 master_main.cc:80] Initializing master server... 04:43:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.907702 5410 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.907796 5410 fs_manager.cc:262] Metadata directory not provided 04:43:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.907812 5410 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.907866 5410 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.907871 5410 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.908545 5410 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data/instance: 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "64a1b6e09890428f9a658c027e5305dc" 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.908771 5410 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/wal/instance: 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "64a1b6e09890428f9a658c027e5305dc" 04:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.910636 5410 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.002s 04:43:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.910791 5410 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.910854 5410 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.911725 5410 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.911751 5410 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/wal 04:43:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "64a1b6e09890428f9a658c027e5305dc" 04:43:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.911794 5410 fs_report.cc:352] FS layout report 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/wal 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/wal 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data/data 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.930904 5410 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.931090 5410 master_main.cc:83] Starting Master server... 04:43:58.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.933877 5410 rpc_server.cc:205] RPC server started. Bound to: 127.5.55.125:49943 04:43:58.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.933935 5410 webserver.cc:175] Starting webserver on 127.5.55.125:0 04:43:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.933943 5410 webserver.cc:186] Document root disabled 04:43:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.934144 5410 webserver.cc:313] Webserver started. Bound to: http://127.5.55.125:53387/ 04:43:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.934273 5410 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-1/data/info.pb 04:43:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.934540 5469 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.935943 5469 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.937948 5469 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.939754 5469 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.940033 5469 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.55.124:37723: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:43:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.943920 5341 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 5410 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.944234 5341 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/wal 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.124:37723 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.124 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/logs 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data/info.pb 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:43:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 with env {} 04:43:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.954185 5472 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.954298 5472 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.954810 5407 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } attempt: 2 04:43:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.955348 5472 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:43:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.955364 5472 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.955369 5472 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:43:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.955375 5472 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.956506 5407 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.956584 5472 master_main.cc:73] Master server non-default flags: 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/wal 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.124:37723 04:43:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data/info.pb 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.124 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.5472 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/logs 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.956691 5407 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.55.124:37723: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:43:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.958230 5472 master_main.cc:80] Initializing master server... 04:43:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.958353 5472 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566194us 04:43:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.958431 5472 fs_manager.cc:262] Metadata directory not provided 04:43:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.958441 5472 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.958475 5472 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.958478 5472 server_base.cc:433] Attempting to create new FS layout instead 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.959090 5472 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data/instance: 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9a918bafcf3641b9a407dd5e14c62e30" 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.959185 5469 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } attempt: 1 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.959312 5472 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/wal/instance: 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9a918bafcf3641b9a407dd5e14c62e30" 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.959455 5469 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.55.124:37723: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:43:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.961246 5472 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.000s 04:43:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.961369 5472 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.961519 5472 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.962412 5472 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.962440 5472 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/wal 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9a918bafcf3641b9a407dd5e14c62e30" 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:58 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.962484 5472 fs_report.cc:352] FS layout report 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/wal 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/wal 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data/data 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:58.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.975831 5472 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:58.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.976009 5472 master_main.cc:83] Starting Master server... 04:43:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.979091 5472 rpc_server.cc:205] RPC server started. Bound to: 127.5.55.124:37723 04:43:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.979248 5472 webserver.cc:175] Starting webserver on 127.5.55.124:0 04:43:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.979341 5472 webserver.cc:186] Document root disabled 04:43:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.979506 5472 webserver.cc:313] Webserver started. Bound to: http://127.5.55.124:55356/ 04:43:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.979625 5472 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/master-2/data/info.pb 04:43:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.979838 5531 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:43:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.981287 5531 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.983328 5531 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.985049 5531 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } has no permanent_uuid. Determining permanent_uuid... 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.986310 5341 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 5472 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.986637 5341 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/wal 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.65:0 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.55.65 04:43:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.65 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/logs 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data/info.pb 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:43:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.988150 5531 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Bootstrap starting. 04:43:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.988400 5531 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: No blocks or log segments found. Creating new log. 04:43:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.988539 5531 log.cc:525] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Log is configured to *not* fsync() on all Append() calls 04:43:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.989621 5531 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: No bootstrap required, opened a new log 04:43:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990597 5531 raft_consensus.cc:325] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990618 5531 raft_consensus.cc:351] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990628 5531 raft_consensus.cc:630] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a918bafcf3641b9a407dd5e14c62e30, State: Initialized, Role: FOLLOWER 04:43:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990684 5531 consensus_queue.cc:228] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990766 5531 sys_catalog.cc:337] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990775 5531 sys_catalog.cc:340] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: This master's current role is: FOLLOWER 04:43:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990826 5531 sys_catalog.cc:424] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: configured and running, proceeding with master startup. 04:43:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.990952 5472 master_main.cc:86] Master server successfully started. 04:43:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.992357 5537 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:43:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:58.992507 5537 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:43:58.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.998733 5536 sys_catalog.cc:337] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:58.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:58.998756 5536 sys_catalog.cc:340] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.000910 5534 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.001034 5534 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.002022 5534 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.002056 5534 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.55.65 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.003288 5534 tablet_server_main.cc:78] Tablet server non-default flags: 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/wal 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.65:0 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data/info.pb 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.65 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.5534 04:43:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.55.65 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/logs 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.005821 5534 tablet_server_main.cc:85] Initializing tablet server... 04:43:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.006079 5534 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566694us 04:43:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.006182 5534 fs_manager.cc:262] Metadata directory not provided 04:43:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.006198 5534 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.006250 5534 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.006258 5534 server_base.cc:433] Attempting to create new FS layout instead 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.007180 5534 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data/instance: 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a6c6364a49584e16a217813b0532ad50" 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.007323 5534 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/wal/instance: 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a6c6364a49584e16a217813b0532ad50" 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.007706 5469 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } attempt: 2 04:43:59.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.010323 5469 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Bootstrap starting. 04:43:59.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.010684 5469 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: No blocks or log segments found. Creating new log. 04:43:59.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.010843 5469 log.cc:525] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Log is configured to *not* fsync() on all Append() calls 04:43:59.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.011072 5534 fs_manager.cc:502] Time spent creating directory manager: real 0.004s user 0.001s sys 0.001s 04:43:59.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.011258 5534 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:59.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.011319 5534 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.012603 5534 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s 04:43:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.012625 5534 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/wal 04:43:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a6c6364a49584e16a217813b0532ad50" 04:43:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.012665 5534 fs_report.cc:352] FS layout report 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/wal 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/wal 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data/data 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.013851 5469 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: No bootstrap required, opened a new log 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015241 5469 raft_consensus.cc:325] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015255 5469 raft_consensus.cc:351] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015262 5469 raft_consensus.cc:630] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64a1b6e09890428f9a658c027e5305dc, State: Initialized, Role: FOLLOWER 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015326 5469 consensus_queue.cc:228] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015403 5469 sys_catalog.cc:337] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015413 5469 sys_catalog.cc:340] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015458 5469 sys_catalog.cc:424] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: configured and running, proceeding with master startup. 04:43:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.015604 5410 master_main.cc:86] Master server successfully started. 04:43:59.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.019866 5407 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } attempt: 1 04:43:59.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.020108 5546 sys_catalog.cc:337] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.020263 5546 sys_catalog.cc:340] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.020512 5547 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:43:59.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.020530 5547 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:43:59.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.022318 5407 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Bootstrap starting. 04:43:59.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.022552 5407 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: No blocks or log segments found. Creating new log. 04:43:59.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.022686 5407 log.cc:525] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Log is configured to *not* fsync() on all Append() calls 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.037739 5407 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: No bootstrap required, opened a new log 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039141 5407 raft_consensus.cc:325] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039155 5407 raft_consensus.cc:351] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039162 5407 raft_consensus.cc:630] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f6f13d895c824d258aab0ab409720547, State: Initialized, Role: FOLLOWER 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039225 5407 consensus_queue.cc:228] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039306 5407 sys_catalog.cc:337] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039314 5407 sys_catalog.cc:340] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039361 5407 sys_catalog.cc:424] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: configured and running, proceeding with master startup. 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.039580 5348 master_main.cc:86] Master server successfully started. 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.040055 5534 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.040428 5554 sys_catalog.cc:337] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.040472 5534 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.040516 5534 tablet_server_main.cc:90] Starting tablet server... 04:43:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.040717 5554 sys_catalog.cc:340] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.043153 5555 catalog_manager.cc:1084] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:43:59.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.043318 5555 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:43:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.047112 5534 rpc_server.cc:205] RPC server started. Bound to: 127.5.55.65:60541 04:43:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.047173 5534 webserver.cc:175] Starting webserver on 127.5.55.65:0 04:43:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.047181 5534 webserver.cc:186] Document root disabled 04:43:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.047420 5534 webserver.cc:313] Webserver started. Bound to: http://127.5.55.65:53533/ 04:43:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.047539 5534 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-0/data/info.pb 04:43:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.048282 5534 tablet_server_main.cc:93] Tablet server successfully started. 04:43:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.050103 5341 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 5534 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.050422 5341 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/wal 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.66:0 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.55.66 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.66 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/logs 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data/info.pb 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.052248 5665 heartbeater.cc:346] Connected to a master server at 127.5.55.126:47505 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.052263 5665 heartbeater.cc:426] Registering TS with master... 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.052301 5665 heartbeater.cc:475] Master 127.5.55.126:47505 requested a full tablet report, sending... 04:43:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.052628 5374 ts_manager.cc:98] Registered new tserver with Master: a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055078 5536 raft_consensus.cc:457] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055130 5536 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Snoozing failure detection for 2.148s (starting election) 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055166 5666 heartbeater.cc:346] Connected to a master server at 127.5.55.125:49943 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055178 5666 heartbeater.cc:426] Registering TS with master... 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055187 5536 raft_consensus.cc:479] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055228 5666 heartbeater.cc:475] Master 127.5.55.125:49943 requested a full tablet report, sending... 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055274 5536 leader_election.cc:251] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055310 5536 leader_election.cc:251] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.055615 5436 ts_manager.cc:98] Registered new tserver with Master: a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056016 5446 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9a918bafcf3641b9a407dd5e14c62e30" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "64a1b6e09890428f9a658c027e5305dc" is_pre_election: true 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056048 5446 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 1.926s (vote granted) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056062 5446 raft_consensus.cc:2255] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a918bafcf3641b9a407dd5e14c62e30 in term 0. 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056166 5384 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9a918bafcf3641b9a407dd5e14c62e30" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f6f13d895c824d258aab0ab409720547" is_pre_election: true 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056174 5481 leader_election.cc:387] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 pre-election: Vote granted by peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056187 5481 leader_election.cc:278] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056191 5384 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.664s (vote granted) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056202 5384 raft_consensus.cc:2255] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a918bafcf3641b9a407dd5e14c62e30 in term 0. 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056248 5536 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Snoozing failure detection for 1.765s (election complete) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056259 5536 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Leader pre-election won for term 1 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056268 5536 raft_consensus.cc:457] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056275 5536 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Snoozing failure detection for 1.975s (starting election) 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056282 5536 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 0 FOLLOWER]: Advancing to term 1 04:43:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056288 5480 leader_election.cc:387] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 pre-election: Vote granted by peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:43:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056849 5536 raft_consensus.cc:479] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056911 5536 leader_election.cc:251] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 election: Requesting vote from peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:43:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.056929 5536 leader_election.cc:251] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 election: Requesting vote from peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:43:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057049 5446 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9a918bafcf3641b9a407dd5e14c62e30" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "64a1b6e09890428f9a658c027e5305dc" 04:43:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057063 5446 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 0 FOLLOWER]: Advancing to term 1 04:43:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057075 5446 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 1.662s (vote granted) 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057459 5384 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9a918bafcf3641b9a407dd5e14c62e30" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f6f13d895c824d258aab0ab409720547" 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057476 5384 raft_consensus.cc:2794] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 0 FOLLOWER]: Advancing to term 1 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057489 5384 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.687s (vote granted) 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057857 5446 raft_consensus.cc:2255] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a918bafcf3641b9a407dd5e14c62e30 in term 1. 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057924 5481 leader_election.cc:387] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 election: Vote granted by peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057934 5481 leader_election.cc:278] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057961 5536 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Snoozing failure detection for 1.708s (election complete) 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057971 5536 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 1 FOLLOWER]: Leader election won for term 1 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.057987 5384 raft_consensus.cc:2255] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a918bafcf3641b9a407dd5e14c62e30 in term 1. 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.058024 5536 raft_consensus.cc:595] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [term 1 LEADER]: Becoming Leader. State: Replica: 9a918bafcf3641b9a407dd5e14c62e30, State: Running, Role: LEADER 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.058079 5536 consensus_queue.cc:205] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.058290 5536 sys_catalog.cc:337] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9a918bafcf3641b9a407dd5e14c62e30. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.058300 5536 sys_catalog.cc:340] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: This master's current role is: LEADER 04:43:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.058363 5480 leader_election.cc:387] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [CANDIDATE]: Term 1 election: Vote granted by peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:43:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.065511 5675 catalog_manager.cc:1026] Loading table and tablet metadata into memory... 04:43:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.065801 5675 catalog_manager.cc:1037] Initializing Kudu internal certificate authority... 04:43:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.068434 5667 heartbeater.cc:346] Connected to a master server at 127.5.55.124:37723 04:43:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.068456 5667 heartbeater.cc:426] Registering TS with master... 04:43:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.068495 5667 heartbeater.cc:475] Master 127.5.55.124:37723 requested a full tablet report, sending... 04:43:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.069591 5498 ts_manager.cc:98] Registered new tserver with Master: a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:43:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.080327 5672 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.080683 5672 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.081784 5672 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.082788 5672 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.55.66 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.084115 5672 tablet_server_main.cc:78] Tablet server non-default flags: 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/wal 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.66:0 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data/info.pb 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.66 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.5672 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.55.66 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/logs 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.092340 5672 tablet_server_main.cc:85] Initializing tablet server... 04:43:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.092717 5672 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566694us 04:43:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.092903 5672 fs_manager.cc:262] Metadata directory not provided 04:43:59.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.093026 5672 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:59.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.093214 5672 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.099162 5672 server_base.cc:433] Attempting to create new FS layout instead 04:43:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.100101 5672 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data/instance: 04:43:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "8842d70b82744db59864d358b1d3a66c" 04:43:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.100594 5672 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/wal/instance: 04:43:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "8842d70b82744db59864d358b1d3a66c" 04:43:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.103679 5672 fs_manager.cc:502] Time spent creating directory manager: real 0.003s user 0.001s sys 0.001s 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.103839 5672 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.103902 5672 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.105530 5672 fs_manager.cc:418] Time spent opening block manager: real 0.001s user 0.001s sys 0.000s 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.105552 5672 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/wal 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "8842d70b82744db59864d358b1d3a66c" 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.105590 5672 fs_report.cc:352] FS layout report 04:43:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/wal 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/wal 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data/data 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.121011 5446 raft_consensus.cc:1083] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Refusing update from remote peer 9a918bafcf3641b9a407dd5e14c62e30: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 04:43:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.121011 5384 raft_consensus.cc:1083] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Refusing update from remote peer 9a918bafcf3641b9a407dd5e14c62e30: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 04:43:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.121428 5536 consensus_queue.cc:963] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [LEADER]: Connected to new peer: Peer: permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:43:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.121556 5536 consensus_queue.cc:963] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:43:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.122112 5546 sys_catalog.cc:337] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9a918bafcf3641b9a407dd5e14c62e30. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.122124 5546 sys_catalog.cc:340] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.122217 5554 sys_catalog.cc:337] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9a918bafcf3641b9a407dd5e14c62e30. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.122231 5554 sys_catalog.cc:340] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.125797 5536 sys_catalog.cc:337] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.125810 5536 sys_catalog.cc:340] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: This master's current role is: LEADER 04:43:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.126291 5546 sys_catalog.cc:337] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.126302 5546 sys_catalog.cc:340] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.129667 5363 tablet.cc:1728] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:43:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.131294 5675 catalog_manager.cc:924] Generated new certificate authority record 04:43:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.131340 5673 sys_catalog.cc:337] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.131350 5673 sys_catalog.cc:340] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30 [sys.catalog]: This master's current role is: LEADER 04:43:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.131567 5554 sys_catalog.cc:337] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9a918bafcf3641b9a407dd5e14c62e30" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:43:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.131580 5554 sys_catalog.cc:340] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: This master's current role is: FOLLOWER 04:43:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.132226 5675 catalog_manager.cc:1046] Loading token signing keys... 04:43:59.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.141182 5675 catalog_manager.cc:4062] T 00000000000000000000000000000000 P 9a918bafcf3641b9a407dd5e14c62e30: Generated new TSK 0 04:43:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.166658 5672 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.167189 5672 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:43:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.167385 5672 tablet_server_main.cc:90] Starting tablet server... 04:43:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.170413 5672 rpc_server.cc:205] RPC server started. Bound to: 127.5.55.66:54897 04:43:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.170574 5672 webserver.cc:175] Starting webserver on 127.5.55.66:0 04:43:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.170675 5672 webserver.cc:186] Document root disabled 04:43:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.170933 5672 webserver.cc:313] Webserver started. Bound to: http://127.5.55.66:54537/ 04:43:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.171178 5672 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-1/data/info.pb 04:43:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.175566 5672 tablet_server_main.cc:93] Tablet server successfully started. 04:43:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.175812 5806 heartbeater.cc:346] Connected to a master server at 127.5.55.124:37723 04:43:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.175833 5806 heartbeater.cc:426] Registering TS with master... 04:43:59.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.175873 5806 heartbeater.cc:475] Master 127.5.55.124:37723 requested a full tablet report, sending... 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.176311 5498 ts_manager.cc:98] Registered new tserver with Master: 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.177567 5498 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.5.55.66:55654 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.177724 5341 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 5672 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.178030 5341 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/wal 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data 04:43:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.67:0 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.55.67 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.67 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/logs 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data/info.pb 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:43:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:43:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.180701 5804 heartbeater.cc:346] Connected to a master server at 127.5.55.126:47505 04:43:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.180817 5804 heartbeater.cc:426] Registering TS with master... 04:43:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.180999 5804 heartbeater.cc:475] Master 127.5.55.126:47505 requested a full tablet report, sending... 04:43:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.180889 5805 heartbeater.cc:346] Connected to a master server at 127.5.55.125:49943 04:43:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.181362 5805 heartbeater.cc:426] Registering TS with master... 04:43:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.181360 5374 ts_manager.cc:98] Registered new tserver with Master: 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:43:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.181609 5805 heartbeater.cc:475] Master 127.5.55.125:49943 requested a full tablet report, sending... 04:43:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.181937 5436 ts_manager.cc:98] Registered new tserver with Master: 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:43:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:43:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.203902 5811 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:43:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.204345 5811 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:43:59.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.205371 5811 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:43:59.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.205502 5811 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.55.67 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.206629 5811 tablet_server_main.cc:78] Tablet server non-default flags: 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/wal 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.55.67:0 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:43:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data/info.pb 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.55.67 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.55.126:47505,127.5.55.125:49943,127.5.55.124:37723 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.5811 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.55.67 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/logs 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:43:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:43:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.208653 5811 tablet_server_main.cc:85] Initializing tablet server... 04:43:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.208897 5811 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 566694us 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.208973 5811 fs_manager.cc:262] Metadata directory not provided 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.208989 5811 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.209043 5811 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.209050 5811 server_base.cc:433] Attempting to create new FS layout instead 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.209733 5811 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data/instance: 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "aba7a3c909364a88bdb41db34d7f7140" 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.209965 5811 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/wal/instance: 04:43:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "aba7a3c909364a88bdb41db34d7f7140" 04:43:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.212126 5811 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.001s 04:43:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.212421 5811 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:43:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.212484 5811 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.213451 5811 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.213477 5811 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/wal 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "aba7a3c909364a88bdb41db34d7f7140" 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:43:59 on quasar-hscjoi-1.vpc.cloudera.com" 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.213516 5811 fs_report.cc:352] FS layout report 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/wal 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/wal 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data/data 04:43:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:43:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:43:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.238435 5811 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:43:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.238662 5811 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:43:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.238725 5811 tablet_server_main.cc:90] Starting tablet server... 04:43:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.245236 5811 rpc_server.cc:205] RPC server started. Bound to: 127.5.55.67:47555 04:43:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.245290 5811 webserver.cc:175] Starting webserver on 127.5.55.67:0 04:43:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.245295 5811 webserver.cc:186] Document root disabled 04:43:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.245461 5811 webserver.cc:313] Webserver started. Bound to: http://127.5.55.67:49527/ 04:43:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.245559 5811 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster7411458279846115255/ts-2/data/info.pb 04:43:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.245796 5811 tablet_server_main.cc:93] Tablet server successfully started. 04:43:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249428 5930 heartbeater.cc:346] Connected to a master server at 127.5.55.126:47505 04:43:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249446 5930 heartbeater.cc:426] Registering TS with master... 04:43:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249487 5930 heartbeater.cc:475] Master 127.5.55.126:47505 requested a full tablet report, sending... 04:43:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249881 5932 heartbeater.cc:346] Connected to a master server at 127.5.55.124:37723 04:43:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249898 5932 heartbeater.cc:426] Registering TS with master... 04:43:59.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249907 5374 ts_manager.cc:98] Registered new tserver with Master: aba7a3c909364a88bdb41db34d7f7140 (127.5.55.67:47555) 04:43:59.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.249929 5932 heartbeater.cc:475] Master 127.5.55.124:37723 requested a full tablet report, sending... 04:43:59.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.250123 5498 ts_manager.cc:98] Registered new tserver with Master: aba7a3c909364a88bdb41db34d7f7140 (127.5.55.67:47555) 04:43:59.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.250900 5498 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.5.55.67:50033 04:43:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.253299 5341 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 5811 04:43:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.253844 5931 heartbeater.cc:346] Connected to a master server at 127.5.55.125:49943 04:43:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.253862 5931 heartbeater.cc:426] Registering TS with master... 04:43:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.253885 5931 heartbeater.cc:475] Master 127.5.55.125:49943 requested a full tablet report, sending... 04:43:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.254273 5436 ts_manager.cc:98] Registered new tserver with Master: aba7a3c909364a88bdb41db34d7f7140 (127.5.55.67:47555) 04:43:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.259440 5341 external_mini_cluster.cc:513] 3 TS(s) registered with all masters 04:43:59.259 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:43:59.260 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_masters { } 04:43:59.262 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.5.55.126" port: 47505 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.5.55.125" port: 49943 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.5.55.124" port: 37723 } } } 04:43:59.263 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_tservers { } 04:43:59.265 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.5.55.65" port: 60541 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.5.55.66" port: 54897 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.5.55.67" port: 47555 } } } 04:43:59.265 [INFO - Test worker] (BaseKuduTest.java:81) Creating a new Kudu client... 04:43:59.275 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:43:59.280 [DEBUG - New I/O worker #28] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.55.125 04:43:59.280 [DEBUG - New I/O worker #30] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.55.124 04:43:59.284 [DEBUG - New I/O worker #29] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.55.126 04:43:59.290 [DEBUG - New I/O worker #28] (Negotiator.java:764) Authenticated connection [id: 0x8fd2dbda, /127.5.55.125:40691 => /127.5.55.125:49943] using SASL/PLAIN 04:43:59.301 [DEBUG - New I/O worker #29] (Negotiator.java:764) Authenticated connection [id: 0xf17059b8, /127.5.55.126:44242 => /127.5.55.126:47505] using SASL/PLAIN 04:43:59.303 [DEBUG - New I/O worker #30] (Negotiator.java:764) Authenticated connection [id: 0x6a14749e, /127.5.55.124:44339 => /127.5.55.124:37723] using SASL/PLAIN 04:43:59.305 [INFO - Time-limited test] (MiniKuduCluster.java:320) Killing master server 127.5.55.124:37723 04:43:59.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:163) Request: stop_daemon { id { type: MASTER index: 2 } } 04:43:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:43:59.307723 5341 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 5472 04:43:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.311292 5549 connection.cc:512] client connection to 127.5.55.124:37723 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.124:37723): Connection reset by peer (error 104) 04:43:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.311295 5683 connection.cc:512] client connection to 127.5.55.124:37723 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.124:37723): Connection reset by peer (error 104) 04:43:59.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:173) Response: 04:43:59.312 [DEBUG - New I/O worker #30] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state READY due to: connection disconnected 04:43:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:43:59.311489 5818 connection.cc:512] client connection to 127.5.55.124:37723 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.124:37723): Connection reset by peer (error 104) 04:43:59.320 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:43:59.320 [INFO - Time-limited test] (BaseKuduTest.java:115) Creating table: TestMasterFailover-killBefore=CREATE_CLIENT 04:43:59.323 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.323 [DEBUG - New I/O boss #36] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.323 [INFO - New I/O boss #36] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.323 [WARN - New I/O boss #36] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.324 [DEBUG - New I/O boss #36] (Deferred.java:1330) callback=retry RPC after error@131145838 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838)), so the following Deferred is getting paused: Deferred@2139701894(state=PAUSED, result=Deferred@1224852605, callback=org.apache.kudu.client.AsyncKuduClient$1@189c0777 -> org.apache.kudu.client.AsyncKuduClient$2@1640865a -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 04:43:59.339 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.339 [DEBUG - New I/O boss #36] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.340 [INFO - New I/O boss #36] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.340 [WARN - New I/O boss #36] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.341 [DEBUG - New I/O boss #36] (Deferred.java:1330) callback=retry RPC after error@1686990611 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611)), so the following Deferred is getting paused: Deferred@483556170(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.360 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.360 [DEBUG - New I/O worker #33] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.361 [INFO - New I/O worker #33] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.361 [WARN - New I/O worker #33] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.361 [DEBUG - New I/O worker #33] (Deferred.java:1330) callback=retry RPC after error@654658160 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160)), so the following Deferred is getting paused: Deferred@700621024(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.379 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.380 [DEBUG - New I/O boss #36] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.380 [INFO - New I/O boss #36] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.381 [WARN - New I/O boss #36] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.381 [DEBUG - New I/O boss #36] (Deferred.java:1330) callback=retry RPC after error@2085003464 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464)), so the following Deferred is getting paused: Deferred@1933679649(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.399 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.399 [DEBUG - New I/O worker #35] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.400 [INFO - New I/O worker #35] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.401 [WARN - New I/O worker #35] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.401 [DEBUG - New I/O worker #35] (Deferred.java:1330) callback=retry RPC after error@1011689783 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783)), so the following Deferred is getting paused: Deferred@847958222(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.439 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.440 [DEBUG - New I/O worker #28] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.440 [INFO - New I/O worker #28] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.440 [WARN - New I/O worker #28] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.441 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=retry RPC after error@1311940599 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599)), so the following Deferred is getting paused: Deferred@1733962202(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.479 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.479 [DEBUG - New I/O worker #29] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.480 [INFO - New I/O worker #29] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.480 [WARN - New I/O worker #29] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.480 [DEBUG - New I/O worker #29] (Deferred.java:1330) callback=retry RPC after error@1887063604 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604)), so the following Deferred is getting paused: Deferred@1950729443(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.579 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.579 [DEBUG - New I/O boss #36] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.580 [INFO - New I/O boss #36] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.580 [WARN - New I/O boss #36] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.580 [DEBUG - New I/O boss #36] (Deferred.java:1330) callback=retry RPC after error@1877943218 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218)), so the following Deferred is getting paused: Deferred@406218205(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:43:59.679 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:43:59.679 [DEBUG - New I/O worker #31] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.680 [INFO - New I/O worker #31] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:43:59.680 [WARN - New I/O worker #31] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:43:59.680 [DEBUG - New I/O worker #31] (Deferred.java:1330) callback=retry RPC after error@1532053463 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463)), so the following Deferred is getting paused: Deferred@1435935437(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:44:00.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.021713 5547 catalog_manager.cc:1082] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: acquiring CA information for follower catalog manager: success 04:44:00.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.021826 5547 catalog_manager.cc:1110] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 04:44:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.044543 5555 catalog_manager.cc:1082] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: acquiring CA information for follower catalog manager: success 04:44:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.044643 5555 catalog_manager.cc:1110] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 04:44:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.073608 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.078 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:00.079 [DEBUG - New I/O worker #32] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:00.079 [INFO - New I/O worker #32] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:00.079 [WARN - New I/O worker #32] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:44:00.079 [DEBUG - New I/O worker #32] (Deferred.java:1330) callback=retry RPC after error@40233111 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111)), so the following Deferred is getting paused: Deferred@211949227(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:44:00.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.178261 5806 heartbeater.cc:467] Master 127.5.55.124:37723 was elected leader, sending a full tablet report... 04:44:00.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.178572 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.251626 5932 heartbeater.cc:467] Master 127.5.55.124:37723 was elected leader, sending a full tablet report... 04:44:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.251924 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642448 5961 raft_consensus.cc:457] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9a918bafcf3641b9a407dd5e14c62e30) 04:44:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642487 5961 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 1.745s (starting election) 04:44:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642545 5961 raft_consensus.cc:479] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642652 5961 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642688 5961 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723) 04:44:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.642930 5417 leader_election.cc:307] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642961 5384 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "64a1b6e09890428f9a658c027e5305dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f6f13d895c824d258aab0ab409720547" is_pre_election: true 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642983 5384 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 2.245s (vote granted) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.642992 5384 raft_consensus.cc:2255] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 64a1b6e09890428f9a658c027e5305dc in term 1. 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643074 5963 raft_consensus.cc:457] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 9a918bafcf3641b9a407dd5e14c62e30) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643084 5418 leader_election.cc:387] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 pre-election: Vote granted by peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643090 5963 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.905s (starting election) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643092 5418 leader_election.cc:278] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643136 5963 raft_consensus.cc:479] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643219 5963 leader_election.cc:251] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643240 5963 leader_election.cc:251] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643349 5446 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f6f13d895c824d258aab0ab409720547" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "64a1b6e09890428f9a658c027e5305dc" is_pre_election: true 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643365 5446 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 2.239s (vote granted) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643374 5446 raft_consensus.cc:2255] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f6f13d895c824d258aab0ab409720547 in term 1. 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.643389 5355 leader_election.cc:307] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643419 5961 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 2.176s (election complete) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643432 5961 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Leader pre-election won for term 2 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643438 5961 raft_consensus.cc:457] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Starting leader election (detected failure of leader 9a918bafcf3641b9a407dd5e14c62e30) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643438 5357 leader_election.cc:387] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 pre-election: Vote granted by peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643446 5357 leader_election.cc:278] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643445 5961 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 1.845s (starting election) 04:44:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643451 5961 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 1 FOLLOWER]: Advancing to term 2 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643486 5963 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.917s (election complete) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643494 5963 raft_consensus.cc:2549] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Leader pre-election won for term 2 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643498 5963 raft_consensus.cc:457] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 9a918bafcf3641b9a407dd5e14c62e30) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643502 5963 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 2.113s (starting election) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643505 5963 raft_consensus.cc:2794] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 1 FOLLOWER]: Advancing to term 2 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643944 5963 raft_consensus.cc:479] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.643982 5963 leader_election.cc:251] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 election: Requesting vote from peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644001 5963 leader_election.cc:251] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 election: Requesting vote from peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644084 5446 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f6f13d895c824d258aab0ab409720547" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "64a1b6e09890428f9a658c027e5305dc" 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.644107 5355 leader_election.cc:307] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644084 5961 raft_consensus.cc:479] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644346 5446 raft_consensus.cc:2185] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate f6f13d895c824d258aab0ab409720547 in current term 2: Already voted for candidate 64a1b6e09890428f9a658c027e5305dc in this term. 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644364 5961 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 election: Requesting vote from peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644382 5961 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 election: Requesting vote from peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644469 5384 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "64a1b6e09890428f9a658c027e5305dc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f6f13d895c824d258aab0ab409720547" 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644482 5384 raft_consensus.cc:2185] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 64a1b6e09890428f9a658c027e5305dc in current term 2: Already voted for candidate f6f13d895c824d258aab0ab409720547 in this term. 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644482 5357 leader_election.cc:401] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 election: Vote denied by peer 64a1b6e09890428f9a658c027e5305dc (127.5.55.125:49943). Message: Invalid argument: T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate f6f13d895c824d258aab0ab409720547 in current term 2: Already voted for candidate 64a1b6e09890428f9a658c027e5305dc in this term. 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644492 5357 leader_election.cc:278] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644531 5963 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.898s (election complete) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644538 5963 raft_consensus.cc:2497] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644546 5418 leader_election.cc:401] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 election: Vote denied by peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505). Message: Invalid argument: T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 64a1b6e09890428f9a658c027e5305dc in current term 2: Already voted for candidate f6f13d895c824d258aab0ab409720547 in this term. 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:00.644608 5417 leader_election.cc:307] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644614 5417 leader_election.cc:278] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644651 5961 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 1.762s (election complete) 04:44:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:00.644662 5961 raft_consensus.cc:2497] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority 04:44:00.820 [DEBUG - New I/O worker #33] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:00.820 [INFO - New I/O worker #33] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:00.820 [WARN - New I/O worker #33] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:44:00.820 [DEBUG - New I/O worker #33] (Deferred.java:1330) callback=retry RPC after error@771563776 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776)), so the following Deferred is getting paused: Deferred@1761585883(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:44:00.820 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:01.074090 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:01.119 [DEBUG - New I/O worker #34] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:01.119 [INFO - New I/O worker #34] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:01.119 [WARN - New I/O worker #34] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:44:01.120 [DEBUG - New I/O worker #34] (Deferred.java:1330) callback=retry RPC after error@402273073 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073)), so the following Deferred is getting paused: Deferred@423481278(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:44:01.120 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:01.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:01.179036 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:01.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:01.252359 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.074537 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.179481 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.252789 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:02.359 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:02.361 [DEBUG - New I/O worker #35] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:02.361 [INFO - New I/O worker #35] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: java.net.ConnectException: Connection refused: /127.5.55.124:37723 04:44:02.361 [WARN - New I/O worker #35] (ConnectToCluster.java:281) Unable to find the leader master 127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723; will retry 04:44:02.361 [DEBUG - New I/O worker #35] (Deferred.java:1330) callback=retry RPC after error@917848647 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073) -> (continuation of Deferred@1941428981 after retry RPC after error@917848647), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073) -> (continuation of Deferred@1941428981 after retry RPC after error@917848647)), so the following Deferred is getting paused: Deferred@1941428981(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:44:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407536 5981 raft_consensus.cc:457] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:44:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407577 5981 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 2.058s (starting election) 04:44:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407634 5981 raft_consensus.cc:479] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407724 5981 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 pre-election: Requesting pre-vote from peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407755 5981 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 pre-election: Requesting pre-vote from peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407922 5384 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "64a1b6e09890428f9a658c027e5305dc" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f6f13d895c824d258aab0ab409720547" is_pre_election: true 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407948 5384 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.766s (vote granted) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.407959 5384 raft_consensus.cc:2255] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 64a1b6e09890428f9a658c027e5305dc in term 2. 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408048 5418 leader_election.cc:387] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 pre-election: Vote granted by peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408059 5418 leader_election.cc:278] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.408085 5417 leader_election.cc:307] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408182 5981 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 2.048s (election complete) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408196 5981 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Leader pre-election won for term 3 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408203 5981 raft_consensus.cc:457] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408211 5981 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 3.170s (starting election) 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408216 5981 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 2 FOLLOWER]: Advancing to term 3 04:44:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408862 5981 raft_consensus.cc:479] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408928 5981 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 election: Requesting vote from peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.408953 5981 leader_election.cc:251] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 election: Requesting vote from peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723) 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409045 5384 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "64a1b6e09890428f9a658c027e5305dc" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f6f13d895c824d258aab0ab409720547" 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409057 5384 raft_consensus.cc:2794] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 2 FOLLOWER]: Advancing to term 3 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409070 5384 raft_consensus.cc:2742] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547: Snoozing failure detection for 1.926s (vote granted) 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.409071 5417 leader_election.cc:307] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409462 5384 raft_consensus.cc:2255] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 64a1b6e09890428f9a658c027e5305dc in term 3. 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409570 5418 leader_election.cc:387] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 election: Vote granted by peer f6f13d895c824d258aab0ab409720547 (127.5.55.126:47505) 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409580 5418 leader_election.cc:278] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409621 5981 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Snoozing failure detection for 3.237s (election complete) 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409633 5981 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 3 FOLLOWER]: Leader election won for term 3 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409683 5981 raft_consensus.cc:595] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [term 3 LEADER]: Becoming Leader. State: Replica: 64a1b6e09890428f9a658c027e5305dc, State: Running, Role: LEADER 04:44:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409735 5981 consensus_queue.cc:205] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409979 5983 sys_catalog.cc:337] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 64a1b6e09890428f9a658c027e5305dc. Latest consensus state: current_term: 3 leader_uuid: "64a1b6e09890428f9a658c027e5305dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.409998 5983 sys_catalog.cc:340] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: This master's current role is: LEADER 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.410125 5985 catalog_manager.cc:1026] Loading table and tablet metadata into memory... 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.410236 5985 catalog_manager.cc:1037] Initializing Kudu internal certificate authority... 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.410429 5985 catalog_manager.cc:1046] Loading token signing keys... 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.410557 5985 catalog_manager.cc:4073] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc: Loaded TSK: 0 04:44:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.410940 5384 raft_consensus.cc:1083] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [term 3 FOLLOWER]: Refusing update from remote peer 64a1b6e09890428f9a658c027e5305dc: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 6. (index mismatch) 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.411000 5417 consensus_peers.cc:433] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc -> Peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Couldn't send request to peer 9a918bafcf3641b9a407dd5e14c62e30 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 1 times. 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411077 5983 consensus_queue.cc:963] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [LEADER]: Connected to new peer: Peer: permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411391 5987 sys_catalog.cc:337] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 64a1b6e09890428f9a658c027e5305dc. Latest consensus state: current_term: 3 leader_uuid: "64a1b6e09890428f9a658c027e5305dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411412 5987 sys_catalog.cc:340] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: This master's current role is: FOLLOWER 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411645 5981 sys_catalog.cc:337] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "64a1b6e09890428f9a658c027e5305dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411661 5981 sys_catalog.cc:340] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc [sys.catalog]: This master's current role is: LEADER 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411837 5987 sys_catalog.cc:337] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "64a1b6e09890428f9a658c027e5305dc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6f13d895c824d258aab0ab409720547" member_type: VOTER last_known_addr { host: "127.5.55.126" port: 47505 } } peers { permanent_uuid: "64a1b6e09890428f9a658c027e5305dc" member_type: VOTER last_known_addr { host: "127.5.55.125" port: 49943 } } peers { permanent_uuid: "9a918bafcf3641b9a407dd5e14c62e30" member_type: VOTER last_known_addr { host: "127.5.55.124" port: 37723 } } } 04:44:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:02.411849 5987 sys_catalog.cc:340] T 00000000000000000000000000000000 P f6f13d895c824d258aab0ab409720547 [sys.catalog]: This master's current role is: FOLLOWER 04:44:02.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:02.923439 5417 consensus_peers.cc:433] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc -> Peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Couldn't send request to peer 9a918bafcf3641b9a407dd5e14c62e30 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 2 times. 04:44:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:03.058663 5436 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.5.55.65:55502 04:44:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:03.074980 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:03.179941 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:03.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:03.253221 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:03.508577 5417 consensus_peers.cc:433] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc -> Peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Couldn't send request to peer 9a918bafcf3641b9a407dd5e14c62e30 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 3 times. 04:44:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.059263 5666 heartbeater.cc:467] Master 127.5.55.125:49943 was elected leader, sending a full tablet report... 04:44:04.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.075408 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.093632 5417 consensus_peers.cc:433] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc -> Peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Couldn't send request to peer 9a918bafcf3641b9a407dd5e14c62e30 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 4 times. 04:44:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.180383 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.184309 5805 heartbeater.cc:467] Master 127.5.55.125:49943 was elected leader, sending a full tablet report... 04:44:04.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.253643 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.256444 5931 heartbeater.cc:467] Master 127.5.55.125:49943 was elected leader, sending a full tablet report... 04:44:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.557276 5417 consensus_peers.cc:433] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc -> Peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Couldn't send request to peer 9a918bafcf3641b9a407dd5e14c62e30 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 5 times. 04:44:04.958 [INFO - New I/O boss #36] (Connection.java:439) Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:04.959 [DEBUG - New I/O boss #36] (Connection.java:688) [peer master-127.5.55.124:37723(127.5.55.124:37723)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:04.959 [INFO - New I/O boss #36] (ConnectToCluster.java:361) Unable to connect to master 127.5.55.124:37723: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723 04:44:04.959 [DEBUG - New I/O worker #28] (AsyncKuduClient.java:1985) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 04:44:04.961 [DEBUG - New I/O worker #28] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=14, DeadlineTracker(timeout=50000, elapsed=5639), Traces: [0ms] querying master, [0ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [0ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [0ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [1ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [0ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [2ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [3ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [16ms] querying master, [16ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [16ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [17ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [17ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [17ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [19ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [20ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [36ms] querying master, [36ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [36ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [37ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [37ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [37ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [39ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [40ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [56ms] querying master, [56ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [56ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [57ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [57ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [57ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [59ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [60ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [76ms] querying master, [76ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [76ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [76ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [76ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [76ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [79ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [80ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [116ms] querying master, [116ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [116ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [116ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [117ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [117ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [119ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [120ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [156ms] querying master, [156ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [156ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [156ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [157ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [157ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [159ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [159ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [256ms] querying master, [256ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [256ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [257ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [257ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [258ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [258ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [259ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.55.124:37723(127.5.55.124:37723): Connection refused: /127.5.55.124:37723, [356ms] querying master, [356ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [356ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [356ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [356ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [356ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [359ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [359ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [755ms] querying master, [755ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [755ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [756ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [756ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [757ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [758ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [758ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [1496ms] querying master, [1496ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [1496ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [1496ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [1497ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [1497ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [1499ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [1499ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [1796ms] querying master, [1796ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [1796ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [1796ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [1797ms] Sub rpc: ConnectToMaster received from server master-127.5.55.126:47505 response OK, [1797ms] Sub rpc: ConnectToMaster received from server master-127.5.55.125:49943 response OK, [1798ms] Sub rpc: ConnectToMaster received from server master-127.5.55.124:37723 response Network error: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [1798ms] delaying RPC due to Service unavailable: Master config (127.5.55.125:49943,127.5.55.126:47505,127.5.55.124:37723) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.55.124:37723, [3036ms] querying master, [3036ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.125:49943, [3036ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.126:47505, [3036ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.55.124:37723, [3039ms] trace too long, truncated, Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073) -> (continuation of Deferred@1941428981 after retry RPC after error@917848647), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073) -> (continuation of Deferred@1941428981 after retry RPC after error@917848647))) after lookup 04:44:04.961 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=retry RPC@2146499514 returned Deferred@1224852605(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073) -> (continuation of Deferred@1941428981 after retry RPC after error@917848647) -> (continuation of Deferred@253865608 after retry RPC@2146499514), errback=(continuation of Deferred@2139701894 after retry RPC after error@131145838) -> (continuation of Deferred@483556170 after retry RPC after error@1686990611) -> (continuation of Deferred@700621024 after retry RPC after error@654658160) -> (continuation of Deferred@1933679649 after retry RPC after error@2085003464) -> (continuation of Deferred@847958222 after retry RPC after error@1011689783) -> (continuation of Deferred@1733962202 after retry RPC after error@1311940599) -> (continuation of Deferred@1950729443 after retry RPC after error@1887063604) -> (continuation of Deferred@406218205 after retry RPC after error@1877943218) -> (continuation of Deferred@1435935437 after retry RPC after error@1532053463) -> (continuation of Deferred@211949227 after retry RPC after error@40233111) -> (continuation of Deferred@1761585883 after retry RPC after error@771563776) -> (continuation of Deferred@423481278 after retry RPC after error@402273073) -> (continuation of Deferred@1941428981 after retry RPC after error@917848647) -> (continuation of Deferred@253865608 after retry RPC@2146499514)), so the following Deferred is getting paused: Deferred@253865608(state=PAUSED, result=Deferred@1224852605, callback=, errback=) 04:44:04.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.963260 5436 catalog_manager.cc:1366] Servicing CreateTable request from {username='jenkins'} at 127.5.55.125:40691: 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "TestMasterFailover-killBefore=CREATE_CLIENT" 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) schema { 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "key" 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: true 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column1_i" 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column2_i" 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column3_s" 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: STRING 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: true 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) encoding: DICT_ENCODING 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) compression: LZ4 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 4096 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column4_b" 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: BOOL 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) partition_schema { 04:44:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) range_schema { 04:44:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "key" 04:44:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.963341 5436 catalog_manager.cc:1504] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table TestMasterFailover-killBefore=CREATE_CLIENT in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 04:44:04.968 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@189c0777@412878711 returned Deferred@1429624252(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@6314b239 -> (continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$1@189c0777@412878711), errback=passthrough -> (continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$1@189c0777@412878711)), so the following Deferred is getting paused: Deferred@2139701894(state=PAUSED, result=Deferred@1429624252, callback=org.apache.kudu.client.AsyncKuduClient$2@1640865a -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 04:44:04.969 [DEBUG - New I/O worker #28] (AsyncKuduClient.java:737) Opened table e317f9d7352f4436b29af6160c52a0a4 04:44:04.969 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450)), so the following Deferred is getting paused: Deferred@2139701894(state=PAUSED, result=Deferred@457291978, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 04:44:04.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.972820 5740 tablet_service.cc:759] Processing CreateTablet for tablet 4206f919a19448cbb6512292de823523 (table=TestMasterFailover-killBefore=CREATE_CLIENT [id=e317f9d7352f4436b29af6160c52a0a4]), partition=RANGE (key) PARTITION UNBOUNDED 04:44:04.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.972914 5740 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 4206f919a19448cbb6512292de823523. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.974457 5867 tablet_service.cc:759] Processing CreateTablet for tablet 4206f919a19448cbb6512292de823523 (table=TestMasterFailover-killBefore=CREATE_CLIENT [id=e317f9d7352f4436b29af6160c52a0a4]), partition=RANGE (key) PARTITION UNBOUNDED 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.974521 5867 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 4206f919a19448cbb6512292de823523. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.974722 5740 ts_tablet_manager.cc:1173] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Registered tablet (data state: TABLET_DATA_READY) 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.974858 6016 ts_tablet_manager.cc:1028] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Bootstrapping tablet 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.974892 6016 tablet_bootstrap.cc:438] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Bootstrap starting. 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.975126 6016 tablet_bootstrap.cc:592] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: No blocks or log segments found. Creating new log. 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.975309 6016 log.cc:525] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Log is configured to *not* fsync() on all Append() calls 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.975735 5602 tablet_service.cc:759] Processing CreateTablet for tablet 4206f919a19448cbb6512292de823523 (table=TestMasterFailover-killBefore=CREATE_CLIENT [id=e317f9d7352f4436b29af6160c52a0a4]), partition=RANGE (key) PARTITION UNBOUNDED 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.975781 5602 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 4206f919a19448cbb6512292de823523. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.976225 5867 ts_tablet_manager.cc:1173] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Registered tablet (data state: TABLET_DATA_READY) 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.976300 6018 ts_tablet_manager.cc:1028] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Bootstrapping tablet 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.976328 6018 tablet_bootstrap.cc:438] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Bootstrap starting. 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.976641 6018 tablet_bootstrap.cc:592] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: No blocks or log segments found. Creating new log. 04:44:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.976789 6018 log.cc:525] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Log is configured to *not* fsync() on all Append() calls 04:44:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.977144 6016 tablet_bootstrap.cc:438] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: No bootstrap required, opened a new log 04:44:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.977171 6016 ts_tablet_manager.cc:1045] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.000s 04:44:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.977259 5602 ts_tablet_manager.cc:1173] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Registered tablet (data state: TABLET_DATA_READY) 04:44:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978348 6016 raft_consensus.cc:325] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978372 6016 raft_consensus.cc:351] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:04.978 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996)), so the following Deferred is getting paused: Deferred@2129280511(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978382 6016 raft_consensus.cc:630] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8842d70b82744db59864d358b1d3a66c, State: Initialized, Role: FOLLOWER 04:44:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978441 6016 consensus_queue.cc:228] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978543 6016 ts_tablet_manager.cc:1073] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s 04:44:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.978895 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978931 6018 tablet_bootstrap.cc:438] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: No bootstrap required, opened a new log 04:44:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.978952 6018 ts_tablet_manager.cc:1045] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.000s 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.980022 5806 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.980267 6018 raft_consensus.cc:325] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.980285 6018 raft_consensus.cc:351] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.980293 6018 raft_consensus.cc:630] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aba7a3c909364a88bdb41db34d7f7140, State: Initialized, Role: FOLLOWER 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.980351 6018 consensus_queue.cc:228] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.980481 6018 ts_tablet_manager.cc:1073] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s 04:44:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.980960 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.981113 6020 ts_tablet_manager.cc:1028] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Bootstrapping tablet 04:44:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.981166 6020 tablet_bootstrap.cc:438] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Bootstrap starting. 04:44:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.981479 6020 tablet_bootstrap.cc:592] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: No blocks or log segments found. Creating new log. 04:44:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.981681 6020 log.cc:525] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Log is configured to *not* fsync() on all Append() calls 04:44:04.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.982094 6020 tablet_bootstrap.cc:438] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: No bootstrap required, opened a new log 04:44:04.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.982120 6020 ts_tablet_manager.cc:1045] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.000s 04:44:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.983218 6020 raft_consensus.cc:325] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.983239 6020 raft_consensus.cc:351] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.983247 6020 raft_consensus.cc:630] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a6c6364a49584e16a217813b0532ad50, State: Initialized, Role: FOLLOWER 04:44:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.983304 6020 consensus_queue.cc:228] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:04.983769 6020 ts_tablet_manager.cc:1073] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s 04:44:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.983887 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.984163 5667 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:04.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.985363 5417 consensus_peers.cc:433] T 00000000000000000000000000000000 P 64a1b6e09890428f9a658c027e5305dc -> Peer 9a918bafcf3641b9a407dd5e14c62e30 (127.5.55.124:37723): Couldn't send request to peer 9a918bafcf3641b9a407dd5e14c62e30 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 6 times. 04:44:04.997 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257)), so the following Deferred is getting paused: Deferred@775468374(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:04.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:04.999449 5933 tablet.cc:1728] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:05.017 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869)), so the following Deferred is getting paused: Deferred@682344639(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:05.037 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302)), so the following Deferred is getting paused: Deferred@895897230(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.052306 5668 tablet.cc:1728] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:05.058 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302) -> (continuation of Deferred@1286828853 after org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302) -> (continuation of Deferred@1286828853 after org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679)), so the following Deferred is getting paused: Deferred@1286828853(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:05.097 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@6c1cd70f@1813829391 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302) -> (continuation of Deferred@1286828853 after org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679) -> (continuation of Deferred@1795810876 after org.apache.kudu.client.AsyncKuduClient$13@6c1cd70f@1813829391), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302) -> (continuation of Deferred@1286828853 after org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679) -> (continuation of Deferred@1795810876 after org.apache.kudu.client.AsyncKuduClient$13@6c1cd70f@1813829391)), so the following Deferred is getting paused: Deferred@1795810876(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:05.138 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@e900d38@244321592 returned Deferred@457291978(state=PENDING, result=null, callback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302) -> (continuation of Deferred@1286828853 after org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679) -> (continuation of Deferred@1795810876 after org.apache.kudu.client.AsyncKuduClient$13@6c1cd70f@1813829391) -> (continuation of Deferred@1500380420 after org.apache.kudu.client.AsyncKuduClient$13@e900d38@244321592), errback=(continuation of Deferred@2139701894 after org.apache.kudu.client.AsyncKuduClient$2@1640865a@373327450) -> (continuation of Deferred@2129280511 after org.apache.kudu.client.AsyncKuduClient$13@199600c4@429260996) -> (continuation of Deferred@775468374 after org.apache.kudu.client.AsyncKuduClient$13@38a9cd61@950652257) -> (continuation of Deferred@682344639 after org.apache.kudu.client.AsyncKuduClient$13@7811075@125898869) -> (continuation of Deferred@895897230 after org.apache.kudu.client.AsyncKuduClient$13@5e9fd20e@1587532302) -> (continuation of Deferred@1286828853 after org.apache.kudu.client.AsyncKuduClient$13@3aebf2ef@988541679) -> (continuation of Deferred@1795810876 after org.apache.kudu.client.AsyncKuduClient$13@6c1cd70f@1813829391) -> (continuation of Deferred@1500380420 after org.apache.kudu.client.AsyncKuduClient$13@e900d38@244321592)), so the following Deferred is getting paused: Deferred@1500380420(state=PAUSED, result=Deferred@457291978, callback=, errback=) 04:44:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.159521 6024 raft_consensus.cc:457] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:44:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.159574 6024 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Snoozing failure detection for 1.541s (starting election) 04:44:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.159634 6024 raft_consensus.cc:479] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.159756 6024 leader_election.cc:251] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:44:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.159786 6024 leader_election.cc:251] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162205 5605 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "4206f919a19448cbb6512292de823523" candidate_uuid: "aba7a3c909364a88bdb41db34d7f7140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a6c6364a49584e16a217813b0532ad50" is_pre_election: true 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162242 5605 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Snoozing failure detection for 2.113s (vote granted) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162255 5605 raft_consensus.cc:2255] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aba7a3c909364a88bdb41db34d7f7140 in term 0. 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162310 5759 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "4206f919a19448cbb6512292de823523" candidate_uuid: "aba7a3c909364a88bdb41db34d7f7140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8842d70b82744db59864d358b1d3a66c" is_pre_election: true 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162336 5759 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Snoozing failure detection for 1.644s (vote granted) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162348 5759 raft_consensus.cc:2255] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aba7a3c909364a88bdb41db34d7f7140 in term 0. 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162487 5819 leader_election.cc:387] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 pre-election: Vote granted by peer 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162498 5819 leader_election.cc:278] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162549 5819 leader_election.cc:387] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 pre-election: Vote granted by peer a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162549 6024 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Snoozing failure detection for 2.032s (election complete) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162564 6024 raft_consensus.cc:2549] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Leader pre-election won for term 1 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162570 6024 raft_consensus.cc:457] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162577 6024 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Snoozing failure detection for 1.941s (starting election) 04:44:05.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.162586 6024 raft_consensus.cc:2794] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 0 FOLLOWER]: Advancing to term 1 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163121 6024 raft_consensus.cc:479] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163188 6024 leader_election.cc:251] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 election: Requesting vote from peer a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163213 6024 leader_election.cc:251] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 election: Requesting vote from peer 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163378 5759 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "4206f919a19448cbb6512292de823523" candidate_uuid: "aba7a3c909364a88bdb41db34d7f7140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8842d70b82744db59864d358b1d3a66c" 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163384 5605 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "4206f919a19448cbb6512292de823523" candidate_uuid: "aba7a3c909364a88bdb41db34d7f7140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a6c6364a49584e16a217813b0532ad50" 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163399 5759 raft_consensus.cc:2794] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [term 0 FOLLOWER]: Advancing to term 1 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163399 5605 raft_consensus.cc:2794] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [term 0 FOLLOWER]: Advancing to term 1 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163411 5605 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50: Snoozing failure detection for 1.823s (vote granted) 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163411 5759 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Snoozing failure detection for 1.712s (vote granted) 04:44:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.163920 5605 raft_consensus.cc:2255] T 4206f919a19448cbb6512292de823523 P a6c6364a49584e16a217813b0532ad50 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aba7a3c909364a88bdb41db34d7f7140 in term 1. 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164049 5819 leader_election.cc:387] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 election: Vote granted by peer a6c6364a49584e16a217813b0532ad50 (127.5.55.65:60541) 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164067 5819 leader_election.cc:278] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164108 6024 raft_consensus.cc:2742] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140: Snoozing failure detection for 1.543s (election complete) 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164120 6024 raft_consensus.cc:2549] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 1 FOLLOWER]: Leader election won for term 1 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164216 6024 raft_consensus.cc:595] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [term 1 LEADER]: Becoming Leader. State: Replica: aba7a3c909364a88bdb41db34d7f7140, State: Running, Role: LEADER 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164269 6024 consensus_queue.cc:205] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } } 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164285 5759 raft_consensus.cc:2255] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aba7a3c909364a88bdb41db34d7f7140 in term 1. 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164432 5819 leader_election.cc:387] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 [CANDIDATE]: Term 1 election: Vote granted by peer 8842d70b82744db59864d358b1d3a66c (127.5.55.66:54897) 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.164513 5932 heartbeater.cc:587] Failed to heartbeat to 127.5.55.124:37723: Network error: Failed to ping master at 127.5.55.124:37723: Client connection negotiation failed: client connection to 127.5.55.124:37723: connect: Connection refused (error 111) 04:44:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.164839 5436 catalog_manager.cc:3758] T 4206f919a19448cbb6512292de823523 P aba7a3c909364a88bdb41db34d7f7140 reported cstate change: term changed from 0 to 1, leader changed from to aba7a3c909364a88bdb41db34d7f7140 (127.5.55.67). New cstate: current_term: 1 leader_uuid: "aba7a3c909364a88bdb41db34d7f7140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a6c6364a49584e16a217813b0532ad50" member_type: VOTER last_known_addr { host: "127.5.55.65" port: 60541 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "aba7a3c909364a88bdb41db34d7f7140" member_type: VOTER last_known_addr { host: "127.5.55.67" port: 47555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8842d70b82744db59864d358b1d3a66c" member_type: VOTER last_known_addr { host: "127.5.55.66" port: 54897 } health_report { overall_health: UNKNOWN } } } 04:44:05.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.179215 5807 tablet.cc:1728] T 4206f919a19448cbb6512292de823523 P 8842d70b82744db59864d358b1d3a66c: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:05.238 [DEBUG - New I/O worker #28] (AsyncKuduClient.java:737) Opened table e317f9d7352f4436b29af6160c52a0a4 04:44:05.240 [DEBUG - New I/O worker #28] (AsyncKuduClient.java:1985) Learned about tablet 4206f919a19448cbb6512292de823523 for table 'TestMasterFailover-killBefore=CREATE_CLIENT' with partition [, ) 04:44:05.240 [DEBUG - New I/O worker #28] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=4206f919a19448cbb6512292de823523@[8842d70b82744db59864d358b1d3a66c(127.5.55.66:54897),a6c6364a49584e16a217813b0532ad50(127.5.55.65:60541),aba7a3c909364a88bdb41db34d7f7140(127.5.55.67:47555)[L]]}] 04:44:05.240 [DEBUG - New I/O worker #28] (AsyncKuduClient.java:1150) Retrying sending RPC ScanRequest(scannerId=null, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=1), Traces: [0ms] querying master, [0ms] Sub rpc: GetTableLocations sending RPC to server master-127.5.55.125:49943, [1ms] Sub rpc: GetTableLocations received from server master-127.5.55.125:49943 response OK, null)) after lookup 04:44:05.241 [DEBUG - New I/O worker #28] (Deferred.java:1330) callback=retry RPC@2002766283 returned Deferred@2032726997(state=PENDING, result=null, callback=(continuation of Deferred@1327215657 after retry RPC@2002766283), errback=(continuation of Deferred@1327215657 after retry RPC@2002766283)), so the following Deferred is getting paused: Deferred@1327215657(state=PAUSED, result=Deferred@2032726997, callback=scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@378b7e6 -> wakeup thread Time-limited test, errback=passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test) 04:44:05.252 [DEBUG - New I/O worker #29] (Negotiator.java:764) Authenticated connection [id: 0x21fda52b, /127.5.55.67:47013 => /127.5.55.67:47555] using TOKEN/null 04:44:05.253 [DEBUG - New I/O worker #29] (AsyncKuduScanner.java:1030) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 0 rows, more = false) for scanner KuduScanner(table=TestMasterFailover-killBefore=CREATE_CLIENT, tablet=4206f919a19448cbb6512292de823523, scannerId=null, scanRequestTimeout=30000) 04:44:05.253 [DEBUG - Time-limited test] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:05.255 [DEBUG - New I/O worker #28] (Connection.java:688) [peer master-127.5.55.125:49943(127.5.55.125:49943)] cleaning up while in state READY due to: connection disconnected 04:44:05.256 [DEBUG - New I/O worker #29] (Connection.java:688) [peer master-127.5.55.126:47505(127.5.55.126:47505)] cleaning up while in state READY due to: connection disconnected 04:44:05.256 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:05.257 [DEBUG - New I/O worker #29] (Connection.java:688) [peer aba7a3c909364a88bdb41db34d7f7140(127.5.55.67:47555)] cleaning up while in state READY due to: connection disconnected 04:44:05.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.258522 5341 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 5534 04:44:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.261263 5355 connection.cc:512] server connection from 127.5.55.65:45587 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.65:45587): Connection reset by peer (error 104) 04:44:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.261332 5418 connection.cc:512] server connection from 127.5.55.65:55502 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.65:55502): Connection reset by peer (error 104) 04:44:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.261394 5819 connection.cc:512] client connection to 127.5.55.65:60541 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.65:60541): Connection reset by peer (error 104) 04:44:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.261492 5341 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 5672 04:44:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.263854 5356 connection.cc:512] server connection from 127.5.55.66:41104 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.66:41104): Connection reset by peer (error 104) 04:44:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.263855 5819 connection.cc:512] client connection to 127.5.55.66:54897 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.66:54897): Connection reset by peer (error 104) 04:44:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.263986 5419 connection.cc:512] server connection from 127.5.55.66:48046 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.66:48046): Connection reset by peer (error 104) 04:44:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.264128 5341 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 5811 04:44:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.266638 5358 connection.cc:512] server connection from 127.5.55.67:54265 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.67:54265): Connection reset by peer (error 104) 04:44:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.266654 5420 connection.cc:512] server connection from 127.5.55.67:54858 recv error: Network error: failed to read from TLS socket (remote: 127.5.55.67:54858): Connection reset by peer (error 104) 04:44:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.266680 5341 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 5348 04:44:05.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.269140 5341 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 5410 04:44:05.276 [INFO - Test worker] (RandomUtils.java:46) Using random seed: 1538221445276 04:44:05.277 [INFO - Test worker] (BaseKuduTest.java:77) Creating a new MiniKuduCluster... 04:44:05.277 [INFO - Test worker] (MiniKuduCluster.java:144) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 04:44:05.277 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 04:44:05.277 [INFO - Test worker] (MiniKuduCluster.java:195) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 04:44:05.278 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415" hms_mode: NONE mini_kdc_options { } } 04:44:05.290 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:05.290 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: start_cluster { } 04:44:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.294823 6035 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/wal 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.254:53107 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.254 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/logs 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data/info.pb 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 with env {} 04:44:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.305047 6042 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.305148 6042 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.305984 6042 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.306002 6042 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.306008 6042 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.306013 6042 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.307106 6042 master_main.cc:73] Master server non-default flags: 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/wal 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.254:53107 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data/info.pb 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.254 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.6042 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/logs 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.309098 6042 master_main.cc:80] Initializing master server... 04:44:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.309273 6042 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 569694us 04:44:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.309371 6042 fs_manager.cc:262] Metadata directory not provided 04:44:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.309387 6042 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.309444 6042 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.309450 6042 server_base.cc:433] Attempting to create new FS layout instead 04:44:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.310101 6042 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data/instance: 04:44:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c7527df67be2426d9a4b7ad626b40060" 04:44:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.310358 6042 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/wal/instance: 04:44:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c7527df67be2426d9a4b7ad626b40060" 04:44:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.312319 6042 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.312474 6042 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.312541 6042 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.313421 6042 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.313448 6042 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/wal 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c7527df67be2426d9a4b7ad626b40060" 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.313494 6042 fs_report.cc:352] FS layout report 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/wal 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/wal 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data/data 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:05.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.339036 6042 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:05.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.339252 6042 master_main.cc:83] Starting Master server... 04:44:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.341989 6042 rpc_server.cc:205] RPC server started. Bound to: 127.5.228.254:53107 04:44:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.342049 6042 webserver.cc:175] Starting webserver on 127.5.228.254:0 04:44:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.342057 6042 webserver.cc:186] Document root disabled 04:44:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.342231 6042 webserver.cc:313] Webserver started. Bound to: http://127.5.228.254:48775/ 04:44:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.342358 6042 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-0/data/info.pb 04:44:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.342660 6101 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.344081 6101 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.346946 6035 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 6042 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.347250 6101 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.347297 6035 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/wal 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.253:51444 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.253 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/logs 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data/info.pb 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 with env {} 04:44:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.348731 6101 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.228.253:51444: Network error: Client connection negotiation failed: client connection to 127.5.228.253:51444: connect: Connection refused (error 111) 04:44:05.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:05.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.358125 6104 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:05.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.358239 6104 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.359197 6104 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.359215 6104 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.359220 6104 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.359226 6104 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.360481 6104 master_main.cc:73] Master server non-default flags: 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/wal 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.253:51444 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data/info.pb 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.253 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.6104 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/logs 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.363405 6104 master_main.cc:80] Initializing master server... 04:44:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.363632 6104 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 569694us 04:44:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.363725 6104 fs_manager.cc:262] Metadata directory not provided 04:44:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.363876 6104 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.364024 6104 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.364107 6104 server_base.cc:433] Attempting to create new FS layout instead 04:44:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.364780 6104 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data/instance: 04:44:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "0d035b3204b043cda7314b9d5911453d" 04:44:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.365015 6104 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/wal/instance: 04:44:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "0d035b3204b043cda7314b9d5911453d" 04:44:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.367214 6104 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.000s 04:44:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.367385 6104 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.367532 6104 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.367905 6101 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } attempt: 1 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.368163 6101 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.228.253:51444: Network error: Client connection negotiation failed: client connection to 127.5.228.253:51444: connect: Connection refused (error 111) 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.368651 6104 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.368677 6104 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/wal 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "0d035b3204b043cda7314b9d5911453d" 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.368721 6104 fs_report.cc:352] FS layout report 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/wal 04:44:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/wal 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data/data 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.392815 6104 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:05.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.393008 6104 master_main.cc:83] Starting Master server... 04:44:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.396236 6104 rpc_server.cc:205] RPC server started. Bound to: 127.5.228.253:51444 04:44:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.396373 6104 webserver.cc:175] Starting webserver on 127.5.228.253:0 04:44:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.396471 6104 webserver.cc:186] Document root disabled 04:44:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.396699 6104 webserver.cc:313] Webserver started. Bound to: http://127.5.228.253:58204/ 04:44:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.396914 6104 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-1/data/info.pb 04:44:05.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.397292 6163 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:05.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.399358 6163 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.400204 6035 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 6104 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.400504 6035 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/wal 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.252:52214 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.252 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/logs 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data/info.pb 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 with env {} 04:44:05.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.407272 6163 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.409843 6163 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.410080 6163 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.228.252:52214: Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:05.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.416296 6101 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } attempt: 2 04:44:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.417695 6165 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.417796 6165 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.418694 6101 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.418848 6165 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.418941 6101 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.228.252:52214: Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.418961 6165 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.418970 6165 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.418977 6165 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.420065 6165 master_main.cc:73] Master server non-default flags: 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/wal 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.252:52214 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data/info.pb 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.252 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.6165 04:44:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/logs 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.421883 6165 master_main.cc:80] Initializing master server... 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422019 6165 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 569694us 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422094 6165 fs_manager.cc:262] Metadata directory not provided 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422104 6165 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422160 6165 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422164 6165 server_base.cc:433] Attempting to create new FS layout instead 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422827 6165 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data/instance: 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "6be64c6316f54c93bf5d4767d465cfa2" 04:44:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.422999 6165 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/wal/instance: 04:44:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "6be64c6316f54c93bf5d4767d465cfa2" 04:44:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.424899 6165 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:05.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.425016 6165 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:05.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.425081 6165 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:05.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.425956 6165 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.425977 6165 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/wal 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "6be64c6316f54c93bf5d4767d465cfa2" 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.426012 6165 fs_report.cc:352] FS layout report 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/wal 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/wal 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data/data 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.429283 6163 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } attempt: 1 04:44:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.429494 6163 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.5.228.252:52214: Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.451050 6165 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.451231 6165 master_main.cc:83] Starting Master server... 04:44:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.454218 6165 rpc_server.cc:205] RPC server started. Bound to: 127.5.228.252:52214 04:44:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.454270 6165 webserver.cc:175] Starting webserver on 127.5.228.252:0 04:44:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.454275 6165 webserver.cc:186] Document root disabled 04:44:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.454407 6165 webserver.cc:313] Webserver started. Bound to: http://127.5.228.252:53523/ 04:44:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.454540 6165 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/master-2/data/info.pb 04:44:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.454639 6035 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 6165 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.455051 6035 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/wal 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.193:0 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.228.193 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.193 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/logs 04:44:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data/info.pb 04:44:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.455826 6226 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.457788 6226 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.460527 6226 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.462493 6226 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } has no permanent_uuid. Determining permanent_uuid... 04:44:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.465844 6226 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Bootstrap starting. 04:44:05.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.466121 6226 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: No blocks or log segments found. Creating new log. 04:44:05.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.466295 6226 log.cc:525] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Log is configured to *not* fsync() on all Append() calls 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.466958 6226 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: No bootstrap required, opened a new log 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468322 6226 raft_consensus.cc:325] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468335 6226 raft_consensus.cc:351] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468343 6226 raft_consensus.cc:630] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6be64c6316f54c93bf5d4767d465cfa2, State: Initialized, Role: FOLLOWER 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.468349 6225 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.468441 6225 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468812 6226 consensus_queue.cc:228] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468909 6226 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468919 6226 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468947 6230 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468961 6230 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.468976 6226 sys_catalog.cc:424] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: configured and running, proceeding with master startup. 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.469373 6225 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.469399 6225 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.228.193 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.469475 6231 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.469486 6231 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.470580 6225 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/wal 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.193:0 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data/info.pb 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.193 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.6225 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.228.193 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/logs 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.471884 6165 master_main.cc:86] Master server successfully started. 04:44:05.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.474531 6225 tablet_server_main.cc:85] Initializing tablet server... 04:44:05.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.474900 6225 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 569694us 04:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.475076 6225 fs_manager.cc:262] Metadata directory not provided 04:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.475194 6225 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.475378 6225 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.475481 6225 server_base.cc:433] Attempting to create new FS layout instead 04:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.476495 6225 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data/instance: 04:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "de45a459704945f087b85d445b0fdbe4" 04:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.476861 6225 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/wal/instance: 04:44:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "de45a459704945f087b85d445b0fdbe4" 04:44:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.477681 6163 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } attempt: 2 04:44:05.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.480518 6163 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Bootstrap starting. 04:44:05.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.480868 6163 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: No blocks or log segments found. Creating new log. 04:44:05.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.481010 6163 log.cc:525] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Log is configured to *not* fsync() on all Append() calls 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.481525 6163 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: No bootstrap required, opened a new log 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.482885 6163 raft_consensus.cc:325] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.482899 6163 raft_consensus.cc:351] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.482908 6163 raft_consensus.cc:630] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0d035b3204b043cda7314b9d5911453d, State: Initialized, Role: FOLLOWER 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.482972 6163 consensus_queue.cc:228] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483094 6163 sys_catalog.cc:337] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483105 6163 sys_catalog.cc:340] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483130 6239 sys_catalog.cc:337] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483144 6239 sys_catalog.cc:340] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483178 6163 sys_catalog.cc:424] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: configured and running, proceeding with master startup. 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483302 6101 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } attempt: 1 04:44:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.483392 6104 master_main.cc:86] Master server successfully started. 04:44:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.483592 6240 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.483603 6240 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.484887 6225 fs_manager.cc:502] Time spent creating directory manager: real 0.008s user 0.001s sys 0.001s 04:44:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.485069 6225 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.485138 6225 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.487083 6101 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Bootstrap starting. 04:44:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.487367 6101 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: No blocks or log segments found. Creating new log. 04:44:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.487504 6101 log.cc:525] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Log is configured to *not* fsync() on all Append() calls 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.488072 6225 fs_manager.cc:418] Time spent opening block manager: real 0.002s user 0.000s sys 0.000s 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.488098 6225 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/wal 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "de45a459704945f087b85d445b0fdbe4" 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.488142 6225 fs_report.cc:352] FS layout report 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/wal 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/wal 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data/data 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.494920 6101 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: No bootstrap required, opened a new log 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496309 6101 raft_consensus.cc:325] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496322 6101 raft_consensus.cc:351] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496330 6101 raft_consensus.cc:630] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c7527df67be2426d9a4b7ad626b40060, State: Initialized, Role: FOLLOWER 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496384 6101 consensus_queue.cc:228] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496459 6101 sys_catalog.cc:337] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496466 6101 sys_catalog.cc:340] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496512 6101 sys_catalog.cc:424] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: configured and running, proceeding with master startup. 04:44:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.496644 6042 master_main.cc:86] Master server successfully started. 04:44:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.501683 6230 raft_consensus.cc:457] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:44:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.501718 6230 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Snoozing failure detection for 1.899s (starting election) 04:44:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.501765 6230 raft_consensus.cc:479] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.501859 6230 leader_election.cc:251] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.501890 6230 leader_election.cc:251] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502112 6078 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6be64c6316f54c93bf5d4767d465cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c7527df67be2426d9a4b7ad626b40060" is_pre_election: true 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502140 6078 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 1.508s (vote granted) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502151 6078 raft_consensus.cc:2255] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6be64c6316f54c93bf5d4767d465cfa2 in term 0. 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502272 6176 leader_election.cc:387] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 pre-election: Vote granted by peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502285 6176 leader_election.cc:278] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502321 6230 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Snoozing failure detection for 2.071s (election complete) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502329 6230 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Leader pre-election won for term 1 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502331 6140 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6be64c6316f54c93bf5d4767d465cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0d035b3204b043cda7314b9d5911453d" is_pre_election: true 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502337 6230 raft_consensus.cc:457] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502344 6230 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Snoozing failure detection for 1.785s (starting election) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502351 6230 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 0 FOLLOWER]: Advancing to term 1 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502355 6140 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 2.197s (vote granted) 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502367 6140 raft_consensus.cc:2255] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6be64c6316f54c93bf5d4767d465cfa2 in term 0. 04:44:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502910 6230 raft_consensus.cc:479] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502967 6230 leader_election.cc:251] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 election: Requesting vote from peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.502982 6230 leader_election.cc:251] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 election: Requesting vote from peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503021 6176 leader_election.cc:387] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 pre-election: Vote granted by peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503124 6078 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6be64c6316f54c93bf5d4767d465cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c7527df67be2426d9a4b7ad626b40060" 04:44:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503163 6078 raft_consensus.cc:2794] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 0 FOLLOWER]: Advancing to term 1 04:44:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503176 6078 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 1.581s (vote granted) 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503648 6078 raft_consensus.cc:2255] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6be64c6316f54c93bf5d4767d465cfa2 in term 1. 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503654 6140 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6be64c6316f54c93bf5d4767d465cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0d035b3204b043cda7314b9d5911453d" 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503669 6140 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 0 FOLLOWER]: Advancing to term 1 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503680 6140 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 1.989s (vote granted) 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503724 6248 sys_catalog.cc:337] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503726 6176 leader_election.cc:387] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 election: Vote granted by peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503736 6176 leader_election.cc:278] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503736 6248 sys_catalog.cc:340] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503765 6230 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Snoozing failure detection for 2.233s (election complete) 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503773 6230 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 1 FOLLOWER]: Leader election won for term 1 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503829 6230 raft_consensus.cc:595] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [term 1 LEADER]: Becoming Leader. State: Replica: 6be64c6316f54c93bf5d4767d465cfa2, State: Running, Role: LEADER 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.503876 6230 consensus_queue.cc:205] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.504058 6230 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6be64c6316f54c93bf5d4767d465cfa2. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.504070 6230 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: This master's current role is: LEADER 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.504186 6140 raft_consensus.cc:2255] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6be64c6316f54c93bf5d4767d465cfa2 in term 1. 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.504264 6176 leader_election.cc:387] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [CANDIDATE]: Term 1 election: Vote granted by peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.504295 6249 catalog_manager.cc:1084] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.504303 6249 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.506176 6252 catalog_manager.cc:1026] Loading table and tablet metadata into memory... 04:44:05.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.506316 6252 catalog_manager.cc:1037] Initializing Kudu internal certificate authority... 04:44:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.540995 6225 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.541468 6225 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.541611 6225 tablet_server_main.cc:90] Starting tablet server... 04:44:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.544915 6225 rpc_server.cc:205] RPC server started. Bound to: 127.5.228.193:55594 04:44:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.544977 6225 webserver.cc:175] Starting webserver on 127.5.228.193:0 04:44:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.544984 6225 webserver.cc:186] Document root disabled 04:44:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.545150 6225 webserver.cc:313] Webserver started. Bound to: http://127.5.228.193:47740/ 04:44:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.545300 6225 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-0/data/info.pb 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.549242 6035 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 6225 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.549384 6225 tablet_server_main.cc:93] Tablet server successfully started. 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.549526 6035 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/wal 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.194:0 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.228.194 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.194 04:44:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/logs 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data/info.pb 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.549641 6363 heartbeater.cc:346] Connected to a master server at 127.5.228.253:51444 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.549655 6363 heartbeater.cc:426] Registering TS with master... 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.549695 6363 heartbeater.cc:475] Master 127.5.228.253:51444 requested a full tablet report, sending... 04:44:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.550004 6130 ts_manager.cc:98] Registered new tserver with Master: de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.558146 6362 heartbeater.cc:346] Connected to a master server at 127.5.228.254:53107 04:44:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.558413 6362 heartbeater.cc:426] Registering TS with master... 04:44:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.558662 6362 heartbeater.cc:475] Master 127.5.228.254:53107 requested a full tablet report, sending... 04:44:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.559145 6068 ts_manager.cc:98] Registered new tserver with Master: de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.559870 6140 raft_consensus.cc:1083] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Refusing update from remote peer 6be64c6316f54c93bf5d4767d465cfa2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 04:44:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.559950 6364 heartbeater.cc:346] Connected to a master server at 127.5.228.252:52214 04:44:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.559984 6364 heartbeater.cc:426] Registering TS with master... 04:44:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.560016 6364 heartbeater.cc:475] Master 127.5.228.252:52214 requested a full tablet report, sending... 04:44:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.560288 6230 consensus_queue.cc:963] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:44:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.560412 6192 ts_manager.cc:98] Registered new tserver with Master: de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.560547 6078 raft_consensus.cc:1083] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Refusing update from remote peer 6be64c6316f54c93bf5d4767d465cfa2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 04:44:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561043 6239 sys_catalog.cc:337] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6be64c6316f54c93bf5d4767d465cfa2. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561058 6239 sys_catalog.cc:340] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561126 6250 consensus_queue.cc:963] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:44:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561801 6248 sys_catalog.cc:337] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6be64c6316f54c93bf5d4767d465cfa2. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561815 6248 sys_catalog.cc:340] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561954 6230 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.561965 6230 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: This master's current role is: LEADER 04:44:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.562172 6239 sys_catalog.cc:337] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.562186 6239 sys_catalog.cc:340] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.562566 6252 catalog_manager.cc:924] Generated new certificate authority record 04:44:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.562943 6230 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.562963 6230 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2 [sys.catalog]: This master's current role is: LEADER 04:44:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.563155 6248 sys_catalog.cc:337] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6be64c6316f54c93bf5d4767d465cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.563169 6248 sys_catalog.cc:340] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: This master's current role is: FOLLOWER 04:44:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.563422 6252 catalog_manager.cc:1046] Loading token signing keys... 04:44:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.567263 6368 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.567656 6368 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.568739 6368 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.568856 6368 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.228.194 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.570222 6368 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/wal 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.194:0 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data/info.pb 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.194 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.6368 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.228.194 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/logs 04:44:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.573958 6368 tablet_server_main.cc:85] Initializing tablet server... 04:44:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.574363 6368 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 569694us 04:44:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.574530 6368 fs_manager.cc:262] Metadata directory not provided 04:44:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.574671 6368 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.574854 6368 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.575134 6368 server_base.cc:433] Attempting to create new FS layout instead 04:44:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.575402 6252 catalog_manager.cc:4062] T 00000000000000000000000000000000 P 6be64c6316f54c93bf5d4767d465cfa2: Generated new TSK 0 04:44:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.576527 6368 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data/instance: 04:44:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "d9e251fe2ca247ebbc73af45e851100b" 04:44:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.576877 6368 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/wal/instance: 04:44:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "d9e251fe2ca247ebbc73af45e851100b" 04:44:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.582270 6368 fs_manager.cc:502] Time spent creating directory manager: real 0.005s user 0.001s sys 0.000s 04:44:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.582429 6368 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.582487 6368 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.583935 6368 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.583956 6368 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/wal 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "d9e251fe2ca247ebbc73af45e851100b" 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.583995 6368 fs_report.cc:352] FS layout report 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/wal 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/wal 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data/data 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.643332 6368 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.643769 6368 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.643906 6368 tablet_server_main.cc:90] Starting tablet server... 04:44:05.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.646788 6368 rpc_server.cc:205] RPC server started. Bound to: 127.5.228.194:49447 04:44:05.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.646944 6368 webserver.cc:175] Starting webserver on 127.5.228.194:0 04:44:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.647042 6368 webserver.cc:186] Document root disabled 04:44:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.647331 6368 webserver.cc:313] Webserver started. Bound to: http://127.5.228.194:37871/ 04:44:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.647547 6368 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-1/data/info.pb 04:44:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.651329 6368 tablet_server_main.cc:93] Tablet server successfully started. 04:44:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.652629 6497 heartbeater.cc:346] Connected to a master server at 127.5.228.253:51444 04:44:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.652647 6497 heartbeater.cc:426] Registering TS with master... 04:44:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.652689 6497 heartbeater.cc:475] Master 127.5.228.253:51444 requested a full tablet report, sending... 04:44:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.653867 6130 ts_manager.cc:98] Registered new tserver with Master: d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.654436 6496 heartbeater.cc:346] Connected to a master server at 127.5.228.254:53107 04:44:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.654611 6496 heartbeater.cc:426] Registering TS with master... 04:44:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.654682 6035 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 6368 04:44:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.654917 6496 heartbeater.cc:475] Master 127.5.228.254:53107 requested a full tablet report, sending... 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.654991 6035 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/wal 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.195:0 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.228.195 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.195 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/logs 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data/info.pb 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.656427 6068 ts_manager.cc:98] Registered new tserver with Master: d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.656452 6498 heartbeater.cc:346] Connected to a master server at 127.5.228.252:52214 04:44:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.656528 6498 heartbeater.cc:426] Registering TS with master... 04:44:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.656561 6498 heartbeater.cc:475] Master 127.5.228.252:52214 requested a full tablet report, sending... 04:44:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.656795 6192 ts_manager.cc:98] Registered new tserver with Master: d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.658015 6192 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.5.228.194:46952 04:44:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.669448 6503 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.669553 6503 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:05.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.670356 6503 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:05.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.670387 6503 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.228.195 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.671416 6503 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/wal 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.5.228.195:0 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data/info.pb 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.5.228.195 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.5.228.254:53107,127.5.228.253:51444,127.5.228.252:52214 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.6503 04:44:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.5.228.195 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/logs 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.673523 6503 tablet_server_main.cc:85] Initializing tablet server... 04:44:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.673756 6503 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 569694us 04:44:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.673884 6503 fs_manager.cc:262] Metadata directory not provided 04:44:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.673904 6503 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.673956 6503 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.673964 6503 server_base.cc:433] Attempting to create new FS layout instead 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.674641 6503 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data/instance: 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.674871 6503 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/wal/instance: 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" 04:44:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.676919 6503 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.002s 04:44:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.677131 6503 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.677244 6503 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.678233 6503 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.678258 6503 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/wal 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:05 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.678297 6503 fs_report.cc:352] FS layout report 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/wal 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/wal 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data/data 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.706887 6503 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.707249 6503 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.707381 6503 tablet_server_main.cc:90] Starting tablet server... 04:44:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.713423 6503 rpc_server.cc:205] RPC server started. Bound to: 127.5.228.195:39769 04:44:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.713480 6503 webserver.cc:175] Starting webserver on 127.5.228.195:0 04:44:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.713487 6503 webserver.cc:186] Document root disabled 04:44:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.713696 6503 webserver.cc:313] Webserver started. Bound to: http://127.5.228.195:52834/ 04:44:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.713832 6503 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3661758396361784415/ts-2/data/info.pb 04:44:05.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.714426 6503 tablet_server_main.cc:93] Tablet server successfully started. 04:44:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.717527 6624 heartbeater.cc:346] Connected to a master server at 127.5.228.252:52214 04:44:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.717545 6624 heartbeater.cc:426] Registering TS with master... 04:44:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.717588 6624 heartbeater.cc:475] Master 127.5.228.252:52214 requested a full tablet report, sending... 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.717999 6192 ts_manager.cc:98] Registered new tserver with Master: 9b5e37d3ecfc4c909c591ab4024b59a8 (127.5.228.195:39769) 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718176 6623 heartbeater.cc:346] Connected to a master server at 127.5.228.253:51444 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718194 6623 heartbeater.cc:426] Registering TS with master... 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718176 6622 heartbeater.cc:346] Connected to a master server at 127.5.228.254:53107 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718216 6622 heartbeater.cc:426] Registering TS with master... 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718225 6623 heartbeater.cc:475] Master 127.5.228.253:51444 requested a full tablet report, sending... 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718240 6622 heartbeater.cc:475] Master 127.5.228.254:53107 requested a full tablet report, sending... 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718516 6068 ts_manager.cc:98] Registered new tserver with Master: 9b5e37d3ecfc4c909c591ab4024b59a8 (127.5.228.195:39769) 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718655 6192 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.5.228.195:57837 04:44:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.718768 6130 ts_manager.cc:98] Registered new tserver with Master: 9b5e37d3ecfc4c909c591ab4024b59a8 (127.5.228.195:39769) 04:44:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.721767 6035 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 6503 04:44:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.727540 6035 external_mini_cluster.cc:513] 3 TS(s) registered with all masters 04:44:05.728 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:05.728 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_masters { } 04:44:05.729 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.5.228.254" port: 53107 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.5.228.253" port: 51444 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.5.228.252" port: 52214 } } } 04:44:05.729 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_tservers { } 04:44:05.729 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.5.228.193" port: 55594 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.5.228.194" port: 49447 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.5.228.195" port: 39769 } } } 04:44:05.730 [INFO - Test worker] (BaseKuduTest.java:81) Creating a new Kudu client... 04:44:05.740 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:05.744 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:05.745 [INFO - Time-limited test] (BaseKuduTest.java:115) Creating table: TestMasterFailover-killBefore=SCAN_TABLE 04:44:05.748 [DEBUG - New I/O worker #46] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.228.252 04:44:05.750 [DEBUG - New I/O worker #47] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.228.254 04:44:05.753 [DEBUG - New I/O worker #48] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.228.253 04:44:05.763 [DEBUG - New I/O worker #47] (Negotiator.java:764) Authenticated connection [id: 0xc8bbdcfa, /127.5.228.254:40288 => /127.5.228.254:53107] using SASL/PLAIN 04:44:05.765 [DEBUG - New I/O worker #48] (Negotiator.java:764) Authenticated connection [id: 0xa328ca24, /127.5.228.253:44088 => /127.5.228.253:51444] using SASL/PLAIN 04:44:05.765 [DEBUG - New I/O worker #46] (Negotiator.java:764) Authenticated connection [id: 0x04efca11, /127.5.228.252:52087 => /127.5.228.252:52214] using SASL/PLAIN 04:44:05.767 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:1985) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 04:44:05.768 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=22), Traces: [0ms] querying master, [1ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [2ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [4ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [19ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [20ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [21ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response OK, null) after lookup 04:44:05.769 [DEBUG - New I/O worker #46] (Deferred.java:1330) callback=retry RPC@39282802 returned Deferred@1196811868(state=PENDING, result=null, callback=(continuation of Deferred@923945639 after retry RPC@39282802), errback=(continuation of Deferred@923945639 after retry RPC@39282802)), so the following Deferred is getting paused: Deferred@923945639(state=PAUSED, result=Deferred@1196811868, callback=org.apache.kudu.client.AsyncKuduClient$1@765883f6 -> org.apache.kudu.client.AsyncKuduClient$2@4ff571b7 -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 04:44:05.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.769938 6192 catalog_manager.cc:1366] Servicing CreateTable request from {username='jenkins'} at 127.5.228.252:52087: 04:44:05.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "TestMasterFailover-killBefore=SCAN_TABLE" 04:44:05.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) schema { 04:44:05.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "key" 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: true 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column1_i" 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column2_i" 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: INT32 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column3_s" 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: STRING 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: true 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) encoding: DICT_ENCODING 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) compression: LZ4 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 4096 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "column4_b" 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) type: BOOL 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_key: false 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) is_nullable: false 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) cfile_block_size: 0 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) partition_schema { 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) range_schema { 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) columns { 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) name: "key" 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) } 04:44:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.770006 6192 catalog_manager.cc:1504] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table TestMasterFailover-killBefore=SCAN_TABLE in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 04:44:05.772 [DEBUG - New I/O worker #46] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@765883f6@1985512438 returned Deferred@662313235(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@2183829 -> (continuation of Deferred@923945639 after org.apache.kudu.client.AsyncKuduClient$1@765883f6@1985512438), errback=passthrough -> (continuation of Deferred@923945639 after org.apache.kudu.client.AsyncKuduClient$1@765883f6@1985512438)), so the following Deferred is getting paused: Deferred@923945639(state=PAUSED, result=Deferred@662313235, callback=org.apache.kudu.client.AsyncKuduClient$2@4ff571b7 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 04:44:05.773 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:737) Opened table 7635bff64aa44ff98c240206270f8835 04:44:05.773 [DEBUG - New I/O worker #46] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@4ff571b7@1341485495 returned Deferred@812473519(state=PENDING, result=null, callback=(continuation of Deferred@923945639 after org.apache.kudu.client.AsyncKuduClient$2@4ff571b7@1341485495), errback=(continuation of Deferred@923945639 after org.apache.kudu.client.AsyncKuduClient$2@4ff571b7@1341485495)), so the following Deferred is getting paused: Deferred@923945639(state=PAUSED, result=Deferred@812473519, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 04:44:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.775645 6299 tablet_service.cc:759] Processing CreateTablet for tablet afbce268bf8c46dfbbccb7356d43b764 (table=TestMasterFailover-killBefore=SCAN_TABLE [id=7635bff64aa44ff98c240206270f8835]), partition=RANGE (key) PARTITION UNBOUNDED 04:44:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.775732 6299 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet afbce268bf8c46dfbbccb7356d43b764. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.775746 6559 tablet_service.cc:759] Processing CreateTablet for tablet afbce268bf8c46dfbbccb7356d43b764 (table=TestMasterFailover-killBefore=SCAN_TABLE [id=7635bff64aa44ff98c240206270f8835]), partition=RANGE (key) PARTITION UNBOUNDED 04:44:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.775815 6559 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet afbce268bf8c46dfbbccb7356d43b764. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777251 6559 ts_tablet_manager.cc:1173] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Registered tablet (data state: TABLET_DATA_READY) 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777504 6655 ts_tablet_manager.cc:1028] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Bootstrapping tablet 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777539 6299 ts_tablet_manager.cc:1173] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Registered tablet (data state: TABLET_DATA_READY) 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777642 6656 ts_tablet_manager.cc:1028] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Bootstrapping tablet 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777679 6655 tablet_bootstrap.cc:438] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Bootstrap starting. 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777844 6656 tablet_bootstrap.cc:438] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Bootstrap starting. 04:44:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.777933 6655 tablet_bootstrap.cc:592] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: No blocks or log segments found. Creating new log. 04:44:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.778192 6655 log.cc:525] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Log is configured to *not* fsync() on all Append() calls 04:44:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.778278 6656 tablet_bootstrap.cc:592] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: No blocks or log segments found. Creating new log. 04:44:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.778425 6656 log.cc:525] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Log is configured to *not* fsync() on all Append() calls 04:44:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.778776 6655 tablet_bootstrap.cc:438] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: No bootstrap required, opened a new log 04:44:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.778802 6655 ts_tablet_manager.cc:1045] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.000s 04:44:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.779762 6655 raft_consensus.cc:325] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.779783 6655 raft_consensus.cc:351] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.779788 6655 raft_consensus.cc:630] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9b5e37d3ecfc4c909c591ab4024b59a8, State: Initialized, Role: FOLLOWER 04:44:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.779850 6655 consensus_queue.cc:228] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.779970 6655 ts_tablet_manager.cc:1073] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s 04:44:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.780122 6624 heartbeater.cc:467] Master 127.5.228.252:52214 was elected leader, sending a full tablet report... 04:44:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.780155 6656 tablet_bootstrap.cc:438] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: No bootstrap required, opened a new log 04:44:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.780176 6656 ts_tablet_manager.cc:1045] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.000s 04:44:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.781777 6656 raft_consensus.cc:325] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.781800 6656 raft_consensus.cc:351] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.781807 6656 raft_consensus.cc:630] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: de45a459704945f087b85d445b0fdbe4, State: Initialized, Role: FOLLOWER 04:44:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.781891 6656 consensus_queue.cc:228] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.782588 6656 ts_tablet_manager.cc:1073] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s 04:44:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.782896 6433 tablet_service.cc:759] Processing CreateTablet for tablet afbce268bf8c46dfbbccb7356d43b764 (table=TestMasterFailover-killBefore=SCAN_TABLE [id=7635bff64aa44ff98c240206270f8835]), partition=RANGE (key) PARTITION UNBOUNDED 04:44:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.782965 6433 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet afbce268bf8c46dfbbccb7356d43b764. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.783041 6192 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.5.228.193:54172 04:44:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.783663 6364 heartbeater.cc:467] Master 127.5.228.252:52214 was elected leader, sending a full tablet report... 04:44:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.784423 6433 ts_tablet_manager.cc:1173] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Registered tablet (data state: TABLET_DATA_READY) 04:44:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.784534 6661 ts_tablet_manager.cc:1028] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Bootstrapping tablet 04:44:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.784703 6661 tablet_bootstrap.cc:438] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Bootstrap starting. 04:44:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.785100 6661 tablet_bootstrap.cc:592] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: No blocks or log segments found. Creating new log. 04:44:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.785329 6661 log.cc:525] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Log is configured to *not* fsync() on all Append() calls 04:44:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.785895 6661 tablet_bootstrap.cc:438] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: No bootstrap required, opened a new log 04:44:05.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.786013 6661 ts_tablet_manager.cc:1045] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.000s 04:44:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.787992 6661 raft_consensus.cc:325] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.788013 6661 raft_consensus.cc:351] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.788022 6661 raft_consensus.cc:630] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9e251fe2ca247ebbc73af45e851100b, State: Initialized, Role: FOLLOWER 04:44:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.788116 6661 consensus_queue.cc:228] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.788204 6498 heartbeater.cc:467] Master 127.5.228.252:52214 was elected leader, sending a full tablet report... 04:44:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.788339 6661 ts_tablet_manager.cc:1073] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s 04:44:05.789 [DEBUG - New I/O worker #46] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@54f1fbdc@1425144796 returned Deferred@812473519(state=PENDING, result=null, callback=(continuation of Deferred@923945639 after org.apache.kudu.client.AsyncKuduClient$2@4ff571b7@1341485495) -> (continuation of Deferred@590660304 after org.apache.kudu.client.AsyncKuduClient$13@54f1fbdc@1425144796), errback=(continuation of Deferred@923945639 after org.apache.kudu.client.AsyncKuduClient$2@4ff571b7@1341485495) -> (continuation of Deferred@590660304 after org.apache.kudu.client.AsyncKuduClient$13@54f1fbdc@1425144796)), so the following Deferred is getting paused: Deferred@590660304(state=PAUSED, result=Deferred@812473519, callback=, errback=) 04:44:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.792518 6659 raft_consensus.cc:457] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:44:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.792551 6659 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Snoozing failure detection for 1.799s (starting election) 04:44:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.792596 6659 raft_consensus.cc:479] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.792701 6659 leader_election.cc:251] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.792728 6659 leader_election.cc:251] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795414 6450 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "afbce268bf8c46dfbbccb7356d43b764" candidate_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9e251fe2ca247ebbc73af45e851100b" is_pre_election: true 04:44:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795449 6450 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Snoozing failure detection for 2.111s (vote granted) 04:44:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795460 6450 raft_consensus.cc:2255] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9b5e37d3ecfc4c909c591ab4024b59a8 in term 0. 04:44:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795800 6513 leader_election.cc:387] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 pre-election: Vote granted by peer d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795822 6513 leader_election.cc:278] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 04:44:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795970 6659 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Snoozing failure detection for 2.131s (election complete) 04:44:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795985 6659 raft_consensus.cc:2549] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Leader pre-election won for term 1 04:44:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.795992 6659 raft_consensus.cc:457] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:44:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796000 6659 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Snoozing failure detection for 2.144s (starting election) 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796008 6659 raft_consensus.cc:2794] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 0 FOLLOWER]: Advancing to term 1 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796598 6659 raft_consensus.cc:479] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796658 6659 leader_election.cc:251] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 election: Requesting vote from peer de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796671 6659 leader_election.cc:251] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 election: Requesting vote from peer d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796919 6316 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "afbce268bf8c46dfbbccb7356d43b764" candidate_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de45a459704945f087b85d445b0fdbe4" is_pre_election: true 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796947 6316 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Snoozing failure detection for 1.804s (vote granted) 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796958 6316 raft_consensus.cc:2255] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9b5e37d3ecfc4c909c591ab4024b59a8 in term 0. 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796978 6450 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "afbce268bf8c46dfbbccb7356d43b764" candidate_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9e251fe2ca247ebbc73af45e851100b" 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.796995 6450 raft_consensus.cc:2794] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 0 FOLLOWER]: Advancing to term 1 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797006 6450 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Snoozing failure detection for 1.666s (vote granted) 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797125 6512 leader_election.cc:387] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 pre-election: Vote granted by peer de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797519 6450 raft_consensus.cc:2255] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9b5e37d3ecfc4c909c591ab4024b59a8 in term 1. 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797674 6312 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "afbce268bf8c46dfbbccb7356d43b764" candidate_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de45a459704945f087b85d445b0fdbe4" 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797688 6513 leader_election.cc:387] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 election: Vote granted by peer d9e251fe2ca247ebbc73af45e851100b (127.5.228.194:49447) 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797693 6312 raft_consensus.cc:2794] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 0 FOLLOWER]: Advancing to term 1 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797700 6513 leader_election.cc:278] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797709 6312 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Snoozing failure detection for 1.714s (vote granted) 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797750 6659 raft_consensus.cc:2742] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Snoozing failure detection for 2.064s (election complete) 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797762 6659 raft_consensus.cc:2549] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 1 FOLLOWER]: Leader election won for term 1 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797822 6659 raft_consensus.cc:595] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [term 1 LEADER]: Becoming Leader. State: Replica: 9b5e37d3ecfc4c909c591ab4024b59a8, State: Running, Role: LEADER 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.797873 6659 consensus_queue.cc:205] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } } 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.798223 6312 raft_consensus.cc:2255] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9b5e37d3ecfc4c909c591ab4024b59a8 in term 1. 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.798471 6512 leader_election.cc:387] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [CANDIDATE]: Term 1 election: Vote granted by peer de45a459704945f087b85d445b0fdbe4 (127.5.228.193:55594) 04:44:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.798701 6191 catalog_manager.cc:3758] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 reported cstate change: term changed from 0 to 1, leader changed from to 9b5e37d3ecfc4c909c591ab4024b59a8 (127.5.228.195). New cstate: current_term: 1 leader_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9b5e37d3ecfc4c909c591ab4024b59a8" member_type: VOTER last_known_addr { host: "127.5.228.195" port: 39769 } health_report { overall_health: HEALTHY } } } 04:44:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.801398 6365 tablet.cc:1728] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:05.808 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:737) Opened table 7635bff64aa44ff98c240206270f8835 04:44:05.809 [INFO - Time-limited test] (MiniKuduCluster.java:320) Killing master server 127.5.228.252:52214 04:44:05.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:163) Request: stop_daemon { id { type: MASTER index: 2 } } 04:44:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:05.812083 6035 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 6165 04:44:05.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.814831 6246 connection.cc:512] client connection to 127.5.228.252:52214 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.252:52214): Connection reset by peer (error 104) 04:44:05.815 [DEBUG - Time-limited test] (MiniKuduCluster.java:173) Response: 04:44:05.815 [DEBUG - New I/O worker #46] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state READY due to: connection disconnected 04:44:05.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.815062 6512 connection.cc:512] client connection to 127.5.228.252:52214 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.252:52214): Connection reset by peer (error 104) 04:44:05.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.815237 6386 connection.cc:512] client connection to 127.5.228.252:52214 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.252:52214): Connection reset by peer (error 104) 04:44:05.818 [DEBUG - New I/O worker #49] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.818 [INFO - New I/O worker #49] (AsyncKuduClient.java:1837) Invalidating location master-127.5.228.252:52214(127.5.228.252:52214) for tablet Kudu Master: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.818 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.829 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.829 [DEBUG - New I/O boss #54] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.829 [INFO - New I/O boss #54] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.829 [WARN - New I/O boss #54] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:05.830 [DEBUG - New I/O boss #54] (Deferred.java:1330) callback=retry RPC after error@619167964 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964)), so the following Deferred is getting paused: Deferred@1817690689(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:05.848 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.848 [DEBUG - New I/O boss #54] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.848 [INFO - New I/O boss #54] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.848 [WARN - New I/O boss #54] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:05.848 [DEBUG - New I/O boss #54] (Deferred.java:1330) callback=retry RPC after error@1005188278 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278)), so the following Deferred is getting paused: Deferred@1901720580(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:05.869 [DEBUG - New I/O worker #52] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.870 [INFO - New I/O worker #52] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.870 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.870 [WARN - New I/O worker #52] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:05.870 [DEBUG - New I/O worker #52] (Deferred.java:1330) callback=retry RPC after error@830190980 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980)), so the following Deferred is getting paused: Deferred@1069305307(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:05.888 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.888 [DEBUG - New I/O boss #54] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.889 [INFO - New I/O boss #54] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.889 [WARN - New I/O boss #54] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:05.889 [DEBUG - New I/O boss #54] (Deferred.java:1330) callback=retry RPC after error@1094517774 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774)), so the following Deferred is getting paused: Deferred@2147099129(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.903417 6499 tablet.cc:1728] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:05.928 [DEBUG - New I/O worker #46] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.928 [INFO - New I/O worker #46] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.928 [WARN - New I/O worker #46] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:05.929 [DEBUG - New I/O worker #46] (Deferred.java:1330) callback=retry RPC after error@1622044655 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655)), so the following Deferred is getting paused: Deferred@989646705(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:05.929 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:05.964632 6625 tablet.cc:1728] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:05.970 [DEBUG - New I/O worker #47] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.970 [INFO - New I/O worker #47] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:05.970 [WARN - New I/O worker #47] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:05.970 [DEBUG - New I/O worker #47] (Deferred.java:1330) callback=retry RPC after error@839701683 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683)), so the following Deferred is getting paused: Deferred@1997425814(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:05.970 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:06.048 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:06.048 [DEBUG - New I/O worker #48] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.049 [INFO - New I/O worker #48] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.049 [WARN - New I/O worker #48] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:06.049 [DEBUG - New I/O worker #48] (Deferred.java:1330) callback=retry RPC after error@1158849070 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070)), so the following Deferred is getting paused: Deferred@1093480020(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:06.149 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:06.150 [DEBUG - New I/O worker #49] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.150 [INFO - New I/O worker #49] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.150 [WARN - New I/O worker #49] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:06.151 [DEBUG - New I/O worker #49] (Deferred.java:1330) callback=retry RPC after error@1091508160 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160)), so the following Deferred is getting paused: Deferred@1338569105(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.184782 6450 raft_consensus.cc:1083] T afbce268bf8c46dfbbccb7356d43b764 P d9e251fe2ca247ebbc73af45e851100b [term 1 FOLLOWER]: Refusing update from remote peer 9b5e37d3ecfc4c909c591ab4024b59a8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 04:44:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.185045 6659 consensus_queue.cc:963] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9e251fe2ca247ebbc73af45e851100b" member_type: VOTER last_known_addr { host: "127.5.228.194" port: 49447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:44:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:06.185655 6498 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.186388 6312 raft_consensus.cc:1083] T afbce268bf8c46dfbbccb7356d43b764 P de45a459704945f087b85d445b0fdbe4 [term 1 FOLLOWER]: Refusing update from remote peer 9b5e37d3ecfc4c909c591ab4024b59a8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 04:44:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:06.186539 6624 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.186789 6668 consensus_queue.cc:963] T afbce268bf8c46dfbbccb7356d43b764 P 9b5e37d3ecfc4c909c591ab4024b59a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "de45a459704945f087b85d445b0fdbe4" member_type: VOTER last_known_addr { host: "127.5.228.193" port: 55594 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 04:44:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:06.186995 6498 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:06.187444 6364 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:06.187721 6364 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:06.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.484760 6240 catalog_manager.cc:1082] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: acquiring CA information for follower catalog manager: success 04:44:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.484897 6240 catalog_manager.cc:1110] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 04:44:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.505436 6249 catalog_manager.cc:1082] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: acquiring CA information for follower catalog manager: success 04:44:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:06.505566 6249 catalog_manager.cc:1110] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 04:44:06.509 [DEBUG - New I/O worker #50] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.509 [INFO - New I/O worker #50] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.509 [WARN - New I/O worker #50] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:06.509 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:06.509 [DEBUG - New I/O worker #50] (Deferred.java:1330) callback=retry RPC after error@1386550779 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779)), so the following Deferred is getting paused: Deferred@1549988319(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:06.528 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:06.528 [DEBUG - New I/O worker #51] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.529 [INFO - New I/O worker #51] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:06.529 [WARN - New I/O worker #51] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:06.529 [DEBUG - New I/O worker #51] (Deferred.java:1330) callback=retry RPC after error@158301524 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524)), so the following Deferred is getting paused: Deferred@128483849(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:06.608872 6624 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.187464 6498 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.188084 6364 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300281 6677 raft_consensus.cc:457] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6be64c6316f54c93bf5d4767d465cfa2) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300318 6677 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 2.080s (starting election) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300377 6677 raft_consensus.cc:479] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300479 6677 leader_election.cc:251] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300626 6677 leader_election.cc:251] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300707 6678 raft_consensus.cc:457] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6be64c6316f54c93bf5d4767d465cfa2) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300729 6678 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 2.114s (starting election) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300736 6140 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c7527df67be2426d9a4b7ad626b40060" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0d035b3204b043cda7314b9d5911453d" is_pre_election: true 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300771 6678 raft_consensus.cc:479] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300813 6140 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 2.207s (vote granted) 04:44:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300825 6140 raft_consensus.cc:2255] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7527df67be2426d9a4b7ad626b40060 in term 1. 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300854 6678 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300936 6052 leader_election.cc:387] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 pre-election: Vote granted by peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300947 6052 leader_election.cc:278] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300972 6078 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d035b3204b043cda7314b9d5911453d" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "c7527df67be2426d9a4b7ad626b40060" is_pre_election: true 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300983 6678 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 pre-election: Requesting pre-vote from peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300987 6078 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 2.013s (vote granted) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.300995 6078 raft_consensus.cc:2255] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0d035b3204b043cda7314b9d5911453d in term 1. 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301015 6677 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 1.632s (election complete) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301028 6677 raft_consensus.cc:2549] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Leader pre-election won for term 2 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301038 6677 raft_consensus.cc:457] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6be64c6316f54c93bf5d4767d465cfa2) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301045 6677 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 1.544s (starting election) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301050 6677 raft_consensus.cc:2794] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 1 FOLLOWER]: Advancing to term 2 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301100 6114 leader_election.cc:387] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 pre-election: Vote granted by peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301110 6114 leader_election.cc:278] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.301110 6051 leader_election.cc:307] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.301201 6113 leader_election.cc:307] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301157 6678 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 1.771s (election complete) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301231 6678 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Leader pre-election won for term 2 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301235 6678 raft_consensus.cc:457] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6be64c6316f54c93bf5d4767d465cfa2) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301240 6678 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 1.731s (starting election) 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301242 6678 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 1 FOLLOWER]: Advancing to term 2 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301651 6678 raft_consensus.cc:479] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301692 6678 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 election: Requesting vote from peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301710 6678 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 election: Requesting vote from peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301764 6677 raft_consensus.cc:479] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301796 6078 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d035b3204b043cda7314b9d5911453d" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "c7527df67be2426d9a4b7ad626b40060" 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301812 6078 raft_consensus.cc:2185] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 0d035b3204b043cda7314b9d5911453d in current term 2: Already voted for candidate c7527df67be2426d9a4b7ad626b40060 in this term. 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301823 6677 leader_election.cc:251] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 election: Requesting vote from peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.301831 6113 leader_election.cc:307] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301842 6677 leader_election.cc:251] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 election: Requesting vote from peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301879 6114 leader_election.cc:401] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 election: Vote denied by peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107). Message: Invalid argument: T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 0d035b3204b043cda7314b9d5911453d in current term 2: Already voted for candidate c7527df67be2426d9a4b7ad626b40060 in this term. 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301889 6114 leader_election.cc:278] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301921 6140 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c7527df67be2426d9a4b7ad626b40060" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0d035b3204b043cda7314b9d5911453d" 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301935 6140 raft_consensus.cc:2185] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c7527df67be2426d9a4b7ad626b40060 in current term 2: Already voted for candidate 0d035b3204b043cda7314b9d5911453d in this term. 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.301998 6678 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 1.649s (election complete) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.302006 6052 leader_election.cc:401] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 election: Vote denied by peer 0d035b3204b043cda7314b9d5911453d (127.5.228.253:51444). Message: Invalid argument: T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c7527df67be2426d9a4b7ad626b40060 in current term 2: Already voted for candidate 0d035b3204b043cda7314b9d5911453d in this term. 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.302007 6678 raft_consensus.cc:2497] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.302023 6051 leader_election.cc:307] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.302034 6051 leader_election.cc:278] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.302078 6677 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 1.996s (election complete) 04:44:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:07.302086 6677 raft_consensus.cc:2497] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority 04:44:07.349 [DEBUG - New I/O worker #52] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:07.349 [INFO - New I/O worker #52] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:07.349 [WARN - New I/O worker #52] (ConnectToCluster.java:281) Unable to find the leader master 127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444; will retry 04:44:07.350 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:07.350 [DEBUG - New I/O worker #52] (Deferred.java:1330) callback=retry RPC after error@998498036 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036)), so the following Deferred is getting paused: Deferred@2122401760(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:07.609495 6624 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:08.187959 6498 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:08.188457 6364 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:08.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:08.609977 6624 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.951845 6686 raft_consensus.cc:457] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.951889 6686 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 2.097s (starting election) 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.951946 6686 raft_consensus.cc:479] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952039 6686 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 pre-election: Requesting pre-vote from peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952070 6686 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 pre-election: Requesting pre-vote from peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214) 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952258 6078 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d035b3204b043cda7314b9d5911453d" candidate_term: 3 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "c7527df67be2426d9a4b7ad626b40060" is_pre_election: true 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952283 6078 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 2.444s (vote granted) 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952296 6078 raft_consensus.cc:2255] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0d035b3204b043cda7314b9d5911453d in term 2. 04:44:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:08.952307 6113 leader_election.cc:307] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952390 6114 leader_election.cc:387] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 pre-election: Vote granted by peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952401 6114 leader_election.cc:278] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952502 6686 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 2.968s (election complete) 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952517 6686 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Leader pre-election won for term 3 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952523 6686 raft_consensus.cc:457] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952533 6686 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 1.799s (starting election) 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.952584 6686 raft_consensus.cc:2794] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 2 FOLLOWER]: Advancing to term 3 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.953783 6686 raft_consensus.cc:479] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.953848 6686 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 election: Requesting vote from peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.953873 6686 leader_election.cc:251] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 election: Requesting vote from peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214) 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.953963 6078 tablet_service.cc:995] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d035b3204b043cda7314b9d5911453d" candidate_term: 3 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "c7527df67be2426d9a4b7ad626b40060" 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.953976 6078 raft_consensus.cc:2794] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 2 FOLLOWER]: Advancing to term 3 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.953987 6078 raft_consensus.cc:2742] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060: Snoozing failure detection for 2.645s (vote granted) 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:08.954020 6113 leader_election.cc:307] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954452 6078 raft_consensus.cc:2255] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0d035b3204b043cda7314b9d5911453d in term 3. 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954541 6114 leader_election.cc:387] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 election: Vote granted by peer c7527df67be2426d9a4b7ad626b40060 (127.5.228.254:53107) 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954551 6114 leader_election.cc:278] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954592 6686 raft_consensus.cc:2742] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Snoozing failure detection for 2.431s (election complete) 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954602 6686 raft_consensus.cc:2549] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 3 FOLLOWER]: Leader election won for term 3 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954643 6686 raft_consensus.cc:595] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [term 3 LEADER]: Becoming Leader. State: Replica: 0d035b3204b043cda7314b9d5911453d, State: Running, Role: LEADER 04:44:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954694 6686 consensus_queue.cc:205] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954937 6688 sys_catalog.cc:337] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0d035b3204b043cda7314b9d5911453d. Latest consensus state: current_term: 3 leader_uuid: "0d035b3204b043cda7314b9d5911453d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.954994 6688 sys_catalog.cc:340] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: This master's current role is: LEADER 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955098 6690 catalog_manager.cc:1026] Loading table and tablet metadata into memory... 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955236 6690 catalog_manager.cc:343] Loaded metadata for table TestMasterFailover-killBefore=SCAN_TABLE [id=7635bff64aa44ff98c240206270f8835] 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955313 6690 catalog_manager.cc:397] Loaded metadata for tablet afbce268bf8c46dfbbccb7356d43b764 (table TestMasterFailover-killBefore=SCAN_TABLE [id=7635bff64aa44ff98c240206270f8835]) 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955332 6690 catalog_manager.cc:1037] Initializing Kudu internal certificate authority... 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955512 6690 catalog_manager.cc:1046] Loading token signing keys... 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955570 6690 catalog_manager.cc:4073] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d: Loaded TSK: 0 04:44:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.955905 6078 raft_consensus.cc:1083] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [term 3 FOLLOWER]: Refusing update from remote peer 0d035b3204b043cda7314b9d5911453d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 3 index: 9. (index mismatch) 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:08.956001 6113 consensus_peers.cc:433] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d -> Peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Couldn't send request to peer 6be64c6316f54c93bf5d4767d465cfa2 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 1 times. 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956035 6688 consensus_queue.cc:963] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956363 6692 sys_catalog.cc:337] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0d035b3204b043cda7314b9d5911453d. Latest consensus state: current_term: 3 leader_uuid: "0d035b3204b043cda7314b9d5911453d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956382 6692 sys_catalog.cc:340] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: This master's current role is: FOLLOWER 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956625 6686 sys_catalog.cc:337] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "0d035b3204b043cda7314b9d5911453d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956640 6686 sys_catalog.cc:340] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d [sys.catalog]: This master's current role is: LEADER 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956883 6692 sys_catalog.cc:337] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "0d035b3204b043cda7314b9d5911453d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c7527df67be2426d9a4b7ad626b40060" member_type: VOTER last_known_addr { host: "127.5.228.254" port: 53107 } } peers { permanent_uuid: "0d035b3204b043cda7314b9d5911453d" member_type: VOTER last_known_addr { host: "127.5.228.253" port: 51444 } } peers { permanent_uuid: "6be64c6316f54c93bf5d4767d465cfa2" member_type: VOTER last_known_addr { host: "127.5.228.252" port: 52214 } } } 04:44:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:08.956899 6692 sys_catalog.cc:340] T 00000000000000000000000000000000 P c7527df67be2426d9a4b7ad626b40060 [sys.catalog]: This master's current role is: FOLLOWER 04:44:09.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.188387 6498 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:09.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.188930 6364 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.505264 6113 consensus_peers.cc:433] T 00000000000000000000000000000000 P 0d035b3204b043cda7314b9d5911453d -> Peer 6be64c6316f54c93bf5d4767d465cfa2 (127.5.228.252:52214): Couldn't send request to peer 6be64c6316f54c93bf5d4767d465cfa2 for tablet 00000000000000000000000000000000. Status: Network error: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111). Retrying in the next heartbeat period. Already tried 2 times. 04:44:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.610436 6624 heartbeater.cc:587] Failed to heartbeat to 127.5.228.252:52214: Network error: Failed to ping master at 127.5.228.252:52214: Client connection negotiation failed: client connection to 127.5.228.252:52214: connect: Connection refused (error 111) 04:44:09.768 [INFO - New I/O boss #54] (Connection.java:439) Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214 04:44:09.769 [DEBUG - New I/O worker #53] (Connection.java:688) [peer master-127.5.228.252:52214(127.5.228.252:52214)] cleaning up while in state CONNECTING due to: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:09.769 [INFO - New I/O worker #53] (ConnectToCluster.java:361) Unable to connect to master 127.5.228.252:52214: java.net.ConnectException: Connection refused: /127.5.228.252:52214 04:44:09.769 [DEBUG - New I/O worker #48] (AsyncKuduClient.java:1985) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 04:44:09.771 [DEBUG - New I/O worker #48] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=GetTableLocations, tablet=Kudu Master, attempt=13, DeadlineTracker(timeout=30000, elapsed=3954), Traces: [0ms] sending RPC to server master-127.5.228.252:52214, [2ms] received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [2ms] delaying RPC due to Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [11ms] querying master, [11ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [11ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [12ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [12ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [13ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [13ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [14ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [30ms] querying master, [30ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [30ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [31ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [32ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [32ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [32ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [32ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [51ms] querying master, [52ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [52ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [52ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [53ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [53ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [54ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [54ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [71ms] querying master, [71ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [71ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [71ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [72ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [72ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [72ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [73ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [110ms] querying master, [110ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [110ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [111ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [111ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [111ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [112ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [113ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [151ms] querying master, [151ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [151ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [152ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [152ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [152ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [154ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [154ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [230ms] querying master, [231ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [231ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [231ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [231ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [231ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [233ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [233ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [331ms] querying master, [331ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [331ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [332ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [332ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [332ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [334ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [334ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [690ms] querying master, [690ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [691ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [691ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [692ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [692ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [693ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [693ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [710ms] querying master, [710ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [710ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [711ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [711ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [711ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [713ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [713ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [1531ms] querying master, [1531ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [1531ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [1531ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [1533ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [1533ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [1533ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [1534ms] delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [3951ms] querying master, [3951ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [3951ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [3951ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [3953ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [3953ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [3953ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036))) after lookup 04:44:09.771 [DEBUG - New I/O worker #48] (Deferred.java:1330) callback=retry RPC@70232085 returned Deferred@552451240(state=PENDING, result=null, callback=get tablet locations from the master for table TestMasterFailover-killBefore=SCAN_TABLE -> release master lookup permit -> retry RPC -> scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036) -> (continuation of Deferred@1706176977 after retry RPC@70232085), errback=passthrough -> release master lookup permit -> retry RPC after error -> passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036) -> (continuation of Deferred@1706176977 after retry RPC@70232085)), so the following Deferred is getting paused: Deferred@1706176977(state=PAUSED, result=Deferred@552451240, callback=, errback=) 04:44:09.772 [DEBUG - New I/O worker #48] (AsyncKuduClient.java:1985) Learned about tablet afbce268bf8c46dfbbccb7356d43b764 for table 'TestMasterFailover-killBefore=SCAN_TABLE' with partition [, ) 04:44:09.772 [DEBUG - New I/O worker #48] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=afbce268bf8c46dfbbccb7356d43b764@[9b5e37d3ecfc4c909c591ab4024b59a8(127.5.228.195:39769)[L],d9e251fe2ca247ebbc73af45e851100b(127.5.228.194:49447),de45a459704945f087b85d445b0fdbe4(127.5.228.193:55594)]}] 04:44:09.773 [DEBUG - New I/O worker #48] (AsyncKuduClient.java:1150) Retrying sending RPC ScanRequest(scannerId=null, attempt=1, KuduRpc(method=Scan, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=3957), Traces: [0ms] querying master, [1ms] Sub rpc: GetTableLocations sending RPC to server master-127.5.228.252:52214, [3ms] Sub rpc: GetTableLocations received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [3ms] Sub rpc: GetTableLocations delaying RPC due to Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [12ms] Sub rpc: GetTableLocations querying master, [12ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [12ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [13ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [13ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [14ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [14ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [15ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [31ms] Sub rpc: GetTableLocations querying master, [31ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [31ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [32ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [33ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [33ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [33ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [33ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [52ms] Sub rpc: GetTableLocations querying master, [53ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [53ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [53ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [54ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [54ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [55ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [55ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [72ms] Sub rpc: GetTableLocations querying master, [72ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [72ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [72ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [73ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [73ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [73ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [74ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.228.252:52214(127.5.228.252:52214): Connection refused: /127.5.228.252:52214, [111ms] Sub rpc: GetTableLocations querying master, [111ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [111ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [112ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [112ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [112ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [113ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [114ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [152ms] Sub rpc: GetTableLocations querying master, [152ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [152ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [153ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [153ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [153ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [155ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [155ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [231ms] Sub rpc: GetTableLocations querying master, [232ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [232ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [232ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [232ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [232ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [234ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [234ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [332ms] Sub rpc: GetTableLocations querying master, [332ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [332ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [333ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [333ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [333ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [335ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [335ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [691ms] Sub rpc: GetTableLocations querying master, [691ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [692ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [692ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [693ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [693ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [694ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [694ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [711ms] Sub rpc: GetTableLocations querying master, [711ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [711ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [712ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [712ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [712ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [714ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [714ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [1532ms] Sub rpc: GetTableLocations querying master, [1532ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [1532ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [1532ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [1534ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [1534ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [1534ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [1535ms] Sub rpc: GetTableLocations delaying RPC due to Service unavailable: Master config (127.5.228.252:52214,127.5.228.254:53107,127.5.228.253:51444) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [3952ms] Sub rpc: GetTableLocations querying master, [3952ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.252:52214, [3952ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.254:53107, [3952ms] Sub rpc: ConnectToMaster sending RPC to server master-127.5.228.253:51444, [3954ms] Sub rpc: ConnectToMaster received from server master-127.5.228.254:53107 response OK, [3954ms] Sub rpc: ConnectToMaster received from server master-127.5.228.252:52214 response Network error: java.net.ConnectException: Connection refused: /127.5.228.252:52214, [3954ms] Sub rpc: ConnectToMaster received from server master-127.5.228.253:51444 response OK, [3956ms] Sub rpc: GetTableLocations sending RPC to server master-127.5.228.253:51444, [3957ms] trace too long, truncated, null)) after lookup 04:44:09.774 [DEBUG - New I/O worker #48] (Deferred.java:1330) callback=retry RPC@487992301 returned Deferred@1677743962(state=PENDING, result=null, callback=(continuation of Deferred@552451240 after retry RPC@487992301), errback=(continuation of Deferred@552451240 after retry RPC@487992301)), so the following Deferred is getting paused: Deferred@552451240(state=PAUSED, result=Deferred@1677743962, callback=scanner opened -> passthrough -> org.apache.kudu.util.ClientTestUtil$2@7517edb2 -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036) -> (continuation of Deferred@1706176977 after retry RPC@70232085), errback=passthrough -> open scanner errback -> org.apache.kudu.util.ClientTestUtil$1@2aa916dc -> wakeup thread Time-limited test -> (continuation of Deferred@1817690689 after retry RPC after error@619167964) -> (continuation of Deferred@1901720580 after retry RPC after error@1005188278) -> (continuation of Deferred@1069305307 after retry RPC after error@830190980) -> (continuation of Deferred@2147099129 after retry RPC after error@1094517774) -> (continuation of Deferred@989646705 after retry RPC after error@1622044655) -> (continuation of Deferred@1997425814 after retry RPC after error@839701683) -> (continuation of Deferred@1093480020 after retry RPC after error@1158849070) -> (continuation of Deferred@1338569105 after retry RPC after error@1091508160) -> (continuation of Deferred@1549988319 after retry RPC after error@1386550779) -> (continuation of Deferred@128483849 after retry RPC after error@158301524) -> (continuation of Deferred@2122401760 after retry RPC after error@998498036) -> (continuation of Deferred@1706176977 after retry RPC@70232085)) 04:44:09.784 [DEBUG - New I/O worker #46] (Negotiator.java:764) Authenticated connection [id: 0x1dfcbb81, /127.5.228.195:57769 => /127.5.228.195:39769] using TOKEN/null 04:44:09.785 [DEBUG - New I/O worker #46] (AsyncKuduScanner.java:1030) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 0 rows, more = false) for scanner KuduScanner(table=TestMasterFailover-killBefore=SCAN_TABLE, tablet=afbce268bf8c46dfbbccb7356d43b764, scannerId=null, scanRequestTimeout=30000) 04:44:09.786 [DEBUG - Time-limited test] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:09.787 [DEBUG - New I/O worker #48] (Connection.java:688) [peer master-127.5.228.253:51444(127.5.228.253:51444)] cleaning up while in state READY due to: connection disconnected 04:44:09.788 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:09.788 [DEBUG - New I/O worker #47] (Connection.java:688) [peer master-127.5.228.254:53107(127.5.228.254:53107)] cleaning up while in state READY due to: connection disconnected 04:44:09.788 [DEBUG - New I/O worker #46] (Connection.java:688) [peer 9b5e37d3ecfc4c909c591ab4024b59a8(127.5.228.195:39769)] cleaning up while in state READY due to: connection disconnected 04:44:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.789743 6035 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 6225 04:44:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.792472 6113 connection.cc:512] server connection from 127.5.228.193:56897 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.193:56897): Connection reset by peer (error 104) 04:44:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.792488 6049 connection.cc:512] server connection from 127.5.228.193:48999 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.193:48999): Connection reset by peer (error 104) 04:44:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.792531 6035 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 6368 04:44:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.792598 6512 connection.cc:512] client connection to 127.5.228.193:55594 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.193:55594): Connection reset by peer (error 104) 04:44:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.794704 6111 connection.cc:512] server connection from 127.5.228.194:45729 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.194:45729): Connection reset by peer (error 104) 04:44:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.794728 6052 connection.cc:512] server connection from 127.5.228.194:56188 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.194:56188): Connection reset by peer (error 104) 04:44:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.794750 6035 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 6503 04:44:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.797247 6049 connection.cc:512] server connection from 127.5.228.195:57193 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.195:57193): Connection reset by peer (error 104) 04:44:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.797291 6035 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 6042 04:44:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.797438 6114 connection.cc:512] server connection from 127.5.228.195:47043 recv error: Network error: failed to read from TLS socket (remote: 127.5.228.195:47043): Connection reset by peer (error 104) 04:44:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.800225 6035 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 6104 04:44:09.807 [INFO - Test worker] (RandomUtils.java:46) Using random seed: 1538221449807 04:44:09.807 [INFO - Test worker] (BaseKuduTest.java:77) Creating a new MiniKuduCluster... 04:44:09.808 [INFO - Test worker] (MiniKuduCluster.java:144) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 04:44:09.808 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 04:44:09.808 [INFO - Test worker] (MiniKuduCluster.java:195) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 04:44:09.809 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898" hms_mode: NONE mini_kdc_options { } } 04:44:09.820 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:09.820 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: start_cluster { } 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.825063 6706 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/wal 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.190:58136 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.190 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/logs 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data/info.pb 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 with env {} 04:44:09.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:09.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.836300 6713 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:09.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.836390 6713 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.837149 6713 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.837167 6713 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.837169 6713 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.837173 6713 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.838472 6713 master_main.cc:73] Master server non-default flags: 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/wal 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.190:58136 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data/info.pb 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.190 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.6713 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/logs 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:09.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.840874 6713 master_main.cc:80] Initializing master server... 04:44:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.841038 6713 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 571694us 04:44:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.841270 6713 fs_manager.cc:262] Metadata directory not provided 04:44:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.841388 6713 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.841544 6713 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.841644 6713 server_base.cc:433] Attempting to create new FS layout instead 04:44:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.842577 6713 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data/instance: 04:44:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c3c2d63a0ada48a6b03692b786a93b08" 04:44:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.843013 6713 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/wal/instance: 04:44:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c3c2d63a0ada48a6b03692b786a93b08" 04:44:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.845866 6713 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.846024 6713 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.846081 6713 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.847556 6713 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.847580 6713 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/wal 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c3c2d63a0ada48a6b03692b786a93b08" 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.847623 6713 fs_report.cc:352] FS layout report 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/wal 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/wal 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data/data 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.885195 6713 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.885394 6713 master_main.cc:83] Starting Master server... 04:44:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.888367 6713 rpc_server.cc:205] RPC server started. Bound to: 127.6.140.190:58136 04:44:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.888496 6713 webserver.cc:175] Starting webserver on 127.6.140.190:0 04:44:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.888507 6713 webserver.cc:186] Document root disabled 04:44:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.888659 6713 webserver.cc:313] Webserver started. Bound to: http://127.6.140.190:59725/ 04:44:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.888787 6713 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-0/data/info.pb 04:44:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.889026 6772 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.891053 6772 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.893612 6772 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.893873 6772 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.6.140.189:38963: Network error: Client connection negotiation failed: client connection to 127.6.140.189:38963: connect: Connection refused (error 111) 04:44:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.895812 6706 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 6713 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.896117 6706 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/wal 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.189:38963 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.189 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/logs 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data/info.pb 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 with env {} 04:44:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.907362 6775 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.907465 6775 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.908316 6775 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.908334 6775 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.908339 6775 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.908345 6775 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.909456 6775 master_main.cc:73] Master server non-default flags: 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/wal 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.189:38963 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data/info.pb 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.189 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.6775 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/logs 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:09.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.911736 6775 master_main.cc:80] Initializing master server... 04:44:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.911984 6775 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 571694us 04:44:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.912194 6775 fs_manager.cc:262] Metadata directory not provided 04:44:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.912300 6775 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.912444 6775 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.912535 6775 server_base.cc:433] Attempting to create new FS layout instead 04:44:09.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.913066 6772 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } attempt: 1 04:44:09.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.913343 6772 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.6.140.189:38963: Network error: Client connection negotiation failed: client connection to 127.6.140.189:38963: connect: Connection refused (error 111) 04:44:09.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.913872 6775 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data/instance: 04:44:09.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" 04:44:09.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.914314 6775 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/wal/instance: 04:44:09.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" 04:44:09.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.916990 6775 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.002s 04:44:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.917188 6775 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.917258 6775 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.918761 6775 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.918789 6775 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/wal 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.918839 6775 fs_report.cc:352] FS layout report 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/wal 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/wal 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data/data 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.952778 6775 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.952960 6775 master_main.cc:83] Starting Master server... 04:44:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.955716 6775 rpc_server.cc:205] RPC server started. Bound to: 127.6.140.189:38963 04:44:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.955770 6775 webserver.cc:175] Starting webserver on 127.6.140.189:0 04:44:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.955773 6775 webserver.cc:186] Document root disabled 04:44:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.955926 6775 webserver.cc:313] Webserver started. Bound to: http://127.6.140.189:55019/ 04:44:09.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.956092 6775 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-1/data/info.pb 04:44:09.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.956369 6834 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.957821 6834 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.958292 6706 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 6775 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.958679 6706 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/wal 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.188:58759 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.188 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/logs 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data/info.pb 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 with env {} 04:44:09.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.960858 6834 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.961535 6772 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } attempt: 2 04:44:09.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.963546 6834 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.963943 6834 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.6.140.188:58759: Network error: Client connection negotiation failed: client connection to 127.6.140.188:58759: connect: Connection refused (error 111) 04:44:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.964303 6772 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.964547 6772 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.6.140.188:58759: Network error: Client connection negotiation failed: client connection to 127.6.140.188:58759: connect: Connection refused (error 111) 04:44:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.970317 6836 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.970417 6836 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.971158 6836 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.971169 6836 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.971173 6836 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.971175 6836 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.972146 6836 master_main.cc:73] Master server non-default flags: 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/wal 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.188:58759 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data/info.pb 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.188 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.6836 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/logs 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.973716 6836 master_main.cc:80] Initializing master server... 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.973836 6836 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 571694us 04:44:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.973906 6836 fs_manager.cc:262] Metadata directory not provided 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.973914 6836 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.973949 6836 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.973953 6836 server_base.cc:433] Attempting to create new FS layout instead 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.974548 6836 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data/instance: 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "3ebb5318abb543d3b18fba6691cca260" 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.974696 6836 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/wal/instance: 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "3ebb5318abb543d3b18fba6691cca260" 04:44:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.976491 6836 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.002s 04:44:09.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.976599 6836 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:09.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.976640 6836 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.977502 6836 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.977524 6836 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/wal 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "3ebb5318abb543d3b18fba6691cca260" 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:09 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.977555 6836 fs_report.cc:352] FS layout report 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/wal 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/wal 04:44:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data/data 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.983130 6834 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } attempt: 1 04:44:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:09.983340 6834 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.6.140.188:58759: Network error: Client connection negotiation failed: client connection to 127.6.140.188:58759: connect: Connection refused (error 111) 04:44:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.988479 6836 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.988646 6836 master_main.cc:83] Starting Master server... 04:44:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.991161 6836 rpc_server.cc:205] RPC server started. Bound to: 127.6.140.188:58759 04:44:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.991241 6836 webserver.cc:175] Starting webserver on 127.6.140.188:0 04:44:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.991250 6836 webserver.cc:186] Document root disabled 04:44:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.991396 6836 webserver.cc:313] Webserver started. Bound to: http://127.6.140.188:50733/ 04:44:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.991565 6836 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/master-2/data/info.pb 04:44:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.991812 6897 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.993250 6897 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.995337 6897 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.996767 6897 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } has no permanent_uuid. Determining permanent_uuid... 04:44:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.999035 6897 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260: Bootstrap starting. 04:44:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.999351 6897 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260: No blocks or log segments found. Creating new log. 04:44:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:09.999557 6897 log.cc:525] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260: Log is configured to *not* fsync() on all Append() calls 04:44:10.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.000213 6897 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260: No bootstrap required, opened a new log 04:44:10.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.000654 6706 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 6836 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.000972 6706 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/wal 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.129:0 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.6.140.129 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.129 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/logs 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data/info.pb 04:44:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002346 6897 raft_consensus.cc:325] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002367 6897 raft_consensus.cc:351] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002377 6897 raft_consensus.cc:630] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3ebb5318abb543d3b18fba6691cca260, State: Initialized, Role: FOLLOWER 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002441 6897 consensus_queue.cc:228] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002689 6897 sys_catalog.cc:337] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } } 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002797 6897 sys_catalog.cc:340] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.002949 6897 sys_catalog.cc:424] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.003243 6836 master_main.cc:86] Master server successfully started. 04:44:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.003290 6903 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.003301 6903 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.015141 6901 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.015307 6901 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.016263 6902 sys_catalog.cc:337] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } } 04:44:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.016290 6902 sys_catalog.cc:340] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.016288 6901 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.016312 6901 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.140.129 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.017578 6901 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/wal 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.129:0 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data/info.pb 04:44:10.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.129 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.6901 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.6.140.129 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/logs 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.020036 6901 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.020337 6901 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.020436 6901 fs_manager.cc:262] Metadata directory not provided 04:44:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.020452 6901 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.020506 6901 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.020514 6901 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.021378 6901 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data/instance: 04:44:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "691a6de3481f4631a8e6e513c1038c8d" 04:44:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.021615 6901 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/wal/instance: 04:44:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "691a6de3481f4631a8e6e513c1038c8d" 04:44:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.024492 6901 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.024657 6901 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.024718 6901 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.026193 6901 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.026240 6901 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/wal 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "691a6de3481f4631a8e6e513c1038c8d" 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.026280 6901 fs_report.cc:352] FS layout report 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/wal 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/wal 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data/data 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.030162 6772 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } attempt: 1 04:44:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.031517 6834 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } attempt: 2 04:44:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.032810 6772 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08: Bootstrap starting. 04:44:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.033197 6772 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08: No blocks or log segments found. Creating new log. 04:44:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.033340 6772 log.cc:525] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08: Log is configured to *not* fsync() on all Append() calls 04:44:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.034426 6834 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9: Bootstrap starting. 04:44:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.034672 6834 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9: No blocks or log segments found. Creating new log. 04:44:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.034809 6834 log.cc:525] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9: Log is configured to *not* fsync() on all Append() calls 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.035639 6834 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9: No bootstrap required, opened a new log 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.036991 6834 raft_consensus.cc:325] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037006 6834 raft_consensus.cc:351] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037014 6834 raft_consensus.cc:630] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9ca74f68e9148e8a37c0c82a1280ea9, State: Initialized, Role: FOLLOWER 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037078 6834 consensus_queue.cc:228] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037155 6834 sys_catalog.cc:337] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } } 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037165 6834 sys_catalog.cc:340] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037241 6834 sys_catalog.cc:424] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.037838 6775 master_main.cc:86] Master server successfully started. 04:44:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.038224 6919 catalog_manager.cc:1084] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.038372 6919 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.038507 6772 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08: No bootstrap required, opened a new log 04:44:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.039860 6772 raft_consensus.cc:325] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } 04:44:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.039883 6772 raft_consensus.cc:351] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.039891 6772 raft_consensus.cc:630] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c3c2d63a0ada48a6b03692b786a93b08, State: Initialized, Role: FOLLOWER 04:44:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.039947 6772 consensus_queue.cc:228] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } 04:44:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.040024 6772 sys_catalog.cc:337] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } } 04:44:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.040035 6772 sys_catalog.cc:340] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.040079 6772 sys_catalog.cc:424] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.040350 6713 master_main.cc:86] Master server successfully started. 04:44:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.040391 6918 sys_catalog.cc:337] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } } 04:44:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.040402 6918 sys_catalog.cc:340] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.046054 6921 sys_catalog.cc:337] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3c2d63a0ada48a6b03692b786a93b08" member_type: VOTER last_known_addr { host: "127.6.140.190" port: 58136 } } peers { permanent_uuid: "e9ca74f68e9148e8a37c0c82a1280ea9" member_type: VOTER last_known_addr { host: "127.6.140.189" port: 38963 } } peers { permanent_uuid: "3ebb5318abb543d3b18fba6691cca260" member_type: VOTER last_known_addr { host: "127.6.140.188" port: 58759 } } } 04:44:10.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.046074 6921 sys_catalog.cc:340] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.046313 6922 catalog_manager.cc:1084] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.046331 6922 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.081296 6901 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.081537 6901 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.081580 6901 tablet_server_main.cc:90] Starting tablet server... 04:44:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.089313 6901 rpc_server.cc:205] RPC server started. Bound to: 127.6.140.129:41267 04:44:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.089499 6901 webserver.cc:175] Starting webserver on 127.6.140.129:0 04:44:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.089598 6901 webserver.cc:186] Document root disabled 04:44:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.089939 6901 webserver.cc:313] Webserver started. Bound to: http://127.6.140.129:45411/ 04:44:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.090180 6901 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-0/data/info.pb 04:44:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.090623 6901 tablet_server_main.cc:93] Tablet server successfully started. 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095067 7032 heartbeater.cc:346] Connected to a master server at 127.6.140.190:58136 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095082 7032 heartbeater.cc:426] Registering TS with master... 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095149 7032 heartbeater.cc:475] Master 127.6.140.190:58136 requested a full tablet report, sending... 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095156 6706 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 6901 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095433 6706 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/wal 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.130:0 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.6.140.130 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.130 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/logs 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data/info.pb 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095579 7034 heartbeater.cc:346] Connected to a master server at 127.6.140.188:58759 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095592 7034 heartbeater.cc:426] Registering TS with master... 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095621 7034 heartbeater.cc:475] Master 127.6.140.188:58759 requested a full tablet report, sending... 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095672 6739 ts_manager.cc:98] Registered new tserver with Master: 691a6de3481f4631a8e6e513c1038c8d (127.6.140.129:41267) 04:44:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.095894 6864 ts_manager.cc:98] Registered new tserver with Master: 691a6de3481f4631a8e6e513c1038c8d (127.6.140.129:41267) 04:44:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.098175 7033 heartbeater.cc:346] Connected to a master server at 127.6.140.189:38963 04:44:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.098194 7033 heartbeater.cc:426] Registering TS with master... 04:44:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.098224 7033 heartbeater.cc:475] Master 127.6.140.189:38963 requested a full tablet report, sending... 04:44:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.098692 6801 ts_manager.cc:98] Registered new tserver with Master: 691a6de3481f4631a8e6e513c1038c8d (127.6.140.129:41267) 04:44:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.110925 7039 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.111050 7039 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.112185 7039 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.112321 7039 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.140.130 04:44:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.113730 7039 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data 04:44:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/wal 04:44:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.130:0 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data/info.pb 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.130 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.7039 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.6.140.130 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/logs 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.117655 7039 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.118086 7039 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.118355 7039 fs_manager.cc:262] Metadata directory not provided 04:44:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.118468 7039 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.118654 7039 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.118744 7039 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.119709 7039 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data/instance: 04:44:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a4cf8ea85f754694a3cd53ea6f507f8e" 04:44:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.120110 7039 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/wal/instance: 04:44:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a4cf8ea85f754694a3cd53ea6f507f8e" 04:44:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.122856 7039 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.002s 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.123013 7039 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.123070 7039 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.124630 7039 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.124651 7039 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/wal 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a4cf8ea85f754694a3cd53ea6f507f8e" 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.124687 7039 fs_report.cc:352] FS layout report 04:44:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/wal 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/wal 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data/data 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.135916 6728 tablet.cc:1728] T 00000000000000000000000000000000 P c3c2d63a0ada48a6b03692b786a93b08: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.168047 7039 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.168321 7039 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.168366 7039 tablet_server_main.cc:90] Starting tablet server... 04:44:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.171447 7039 rpc_server.cc:205] RPC server started. Bound to: 127.6.140.130:41083 04:44:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.171509 7039 webserver.cc:175] Starting webserver on 127.6.140.130:0 04:44:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.171515 7039 webserver.cc:186] Document root disabled 04:44:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.171685 7039 webserver.cc:313] Webserver started. Bound to: http://127.6.140.130:36141/ 04:44:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.171934 7039 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-1/data/info.pb 04:44:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.176297 7039 tablet_server_main.cc:93] Tablet server successfully started. 04:44:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.177012 7159 heartbeater.cc:346] Connected to a master server at 127.6.140.189:38963 04:44:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.177027 7159 heartbeater.cc:426] Registering TS with master... 04:44:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.177067 7159 heartbeater.cc:475] Master 127.6.140.189:38963 requested a full tablet report, sending... 04:44:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.177757 6801 ts_manager.cc:98] Registered new tserver with Master: a4cf8ea85f754694a3cd53ea6f507f8e (127.6.140.130:41083) 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.180115 6706 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 7039 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.180449 6706 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/wal 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.131:0 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.6.140.131 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.131 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/logs 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data/info.pb 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.182317 7158 heartbeater.cc:346] Connected to a master server at 127.6.140.190:58136 04:44:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.182332 7158 heartbeater.cc:426] Registering TS with master... 04:44:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.182361 7158 heartbeater.cc:475] Master 127.6.140.190:58136 requested a full tablet report, sending... 04:44:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.182610 6739 ts_manager.cc:98] Registered new tserver with Master: a4cf8ea85f754694a3cd53ea6f507f8e (127.6.140.130:41083) 04:44:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.183487 7160 heartbeater.cc:346] Connected to a master server at 127.6.140.188:58759 04:44:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.183512 7160 heartbeater.cc:426] Registering TS with master... 04:44:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.183547 7160 heartbeater.cc:475] Master 127.6.140.188:58759 requested a full tablet report, sending... 04:44:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.183931 6864 ts_manager.cc:98] Registered new tserver with Master: a4cf8ea85f754694a3cd53ea6f507f8e (127.6.140.130:41083) 04:44:10.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.195369 7165 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.195500 7165 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.196475 7165 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.196511 7165 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.140.131 04:44:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.197824 7165 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data 04:44:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/wal 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.6.140.131:0 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data/info.pb 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.6.140.131 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.7165 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.6.140.131 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/logs 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.200284 7165 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.200546 7165 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.200628 7165 fs_manager.cc:262] Metadata directory not provided 04:44:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.200645 7165 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.200698 7165 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.200706 7165 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.201541 7165 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data/instance: 04:44:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a365b1855e9c46bba11822953efdfe02" 04:44:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.201786 7165 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/wal/instance: 04:44:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a365b1855e9c46bba11822953efdfe02" 04:44:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.203246 6790 tablet.cc:1728] T 00000000000000000000000000000000 P e9ca74f68e9148e8a37c0c82a1280ea9: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.204437 7165 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.204607 7165 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.204679 7165 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.205888 7165 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.205916 7165 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/wal 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a365b1855e9c46bba11822953efdfe02" 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.205955 7165 fs_report.cc:352] FS layout report 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/wal 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/wal 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data/data 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.227303 7165 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.227545 7165 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.227592 7165 tablet_server_main.cc:90] Starting tablet server... 04:44:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.234233 7165 rpc_server.cc:205] RPC server started. Bound to: 127.6.140.131:40979 04:44:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.234362 7165 webserver.cc:175] Starting webserver on 127.6.140.131:0 04:44:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.234375 7165 webserver.cc:186] Document root disabled 04:44:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.234580 7165 webserver.cc:313] Webserver started. Bound to: http://127.6.140.131:50252/ 04:44:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.234714 7165 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6731264114035472898/ts-2/data/info.pb 04:44:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.235213 7165 tablet_server_main.cc:93] Tablet server successfully started. 04:44:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.238862 6853 tablet.cc:1728] T 00000000000000000000000000000000 P 3ebb5318abb543d3b18fba6691cca260: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.239439 7285 heartbeater.cc:346] Connected to a master server at 127.6.140.189:38963 04:44:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.239454 7285 heartbeater.cc:426] Registering TS with master... 04:44:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.239495 7285 heartbeater.cc:475] Master 127.6.140.189:38963 requested a full tablet report, sending... 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.239912 6801 ts_manager.cc:98] Registered new tserver with Master: a365b1855e9c46bba11822953efdfe02 (127.6.140.131:40979) 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.239962 7284 heartbeater.cc:346] Connected to a master server at 127.6.140.190:58136 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.239975 7284 heartbeater.cc:426] Registering TS with master... 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.240003 7284 heartbeater.cc:475] Master 127.6.140.190:58136 requested a full tablet report, sending... 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.240522 6739 ts_manager.cc:98] Registered new tserver with Master: a365b1855e9c46bba11822953efdfe02 (127.6.140.131:40979) 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.240614 7286 heartbeater.cc:346] Connected to a master server at 127.6.140.188:58759 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.240629 7286 heartbeater.cc:426] Registering TS with master... 04:44:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.240659 7286 heartbeater.cc:475] Master 127.6.140.188:58759 requested a full tablet report, sending... 04:44:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.240885 6864 ts_manager.cc:98] Registered new tserver with Master: a365b1855e9c46bba11822953efdfe02 (127.6.140.131:40979) 04:44:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.242632 6706 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 7165 04:44:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.248144 6706 external_mini_cluster.cc:513] 3 TS(s) registered with all masters 04:44:10.248 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:10.248 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_masters { } 04:44:10.249 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.6.140.190" port: 58136 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.6.140.189" port: 38963 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.6.140.188" port: 58759 } } } 04:44:10.249 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_tservers { } 04:44:10.250 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.6.140.129" port: 41267 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.6.140.130" port: 41083 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.6.140.131" port: 40979 } } } 04:44:10.250 [INFO - Test worker] (BaseKuduTest.java:81) Creating a new Kudu client... 04:44:10.259 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:10.263 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:10.266 [DEBUG - New I/O worker #64] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.6.140.190 04:44:10.267 [DEBUG - New I/O worker #65] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.6.140.189 04:44:10.269 [DEBUG - New I/O worker #66] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.6.140.188 04:44:10.275 [DEBUG - New I/O worker #64] (Negotiator.java:764) Authenticated connection [id: 0x5eb94f1d, /127.6.140.190:47214 => /127.6.140.190:58136] using SASL/PLAIN 04:44:10.280 [DEBUG - New I/O worker #66] (Negotiator.java:764) Authenticated connection [id: 0xdcf98880, /127.6.140.188:45624 => /127.6.140.188:58759] using SASL/PLAIN 04:44:10.281 [DEBUG - New I/O worker #65] (Negotiator.java:764) Authenticated connection [id: 0x0b9c6494, /127.6.140.189:58272 => /127.6.140.189:38963] using SASL/PLAIN 04:44:10.282 [WARN - New I/O worker #65] (ConnectToCluster.java:278) None of the provided masters 127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759 is a leader; will retry 04:44:10.283 [DEBUG - Time-limited test] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:10.285 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:10.285 [DEBUG - New I/O worker #66] (Connection.java:688) [peer master-127.6.140.188:58759(127.6.140.188:58759)] cleaning up while in state READY due to: connection disconnected 04:44:10.286 [DEBUG - New I/O worker #64] (Connection.java:688) [peer master-127.6.140.190:58136(127.6.140.190:58136)] cleaning up while in state READY due to: connection disconnected 04:44:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.287370 6706 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 6901 04:44:10.285 [DEBUG - New I/O worker #65] (Connection.java:688) [peer master-127.6.140.189:38963(127.6.140.189:38963)] cleaning up while in state READY due to: connection disconnected 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.289587 6848 connection.cc:512] server connection from 127.6.140.129:45039 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.129:45039): Connection reset by peer (error 104) 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.289595 6785 connection.cc:512] server connection from 127.6.140.129:53870 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.129:53870): Connection reset by peer (error 104) 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.289628 6706 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 7039 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.289713 6722 connection.cc:512] server connection from 127.6.140.129:37723 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.129:37723): Connection reset by peer (error 104) 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.291640 6783 connection.cc:512] server connection from 127.6.140.130:60025 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.130:60025): Connection reset by peer (error 104) 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.291651 6846 connection.cc:512] server connection from 127.6.140.130:35596 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.130:35596): Connection reset by peer (error 104) 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.291744 6720 connection.cc:512] server connection from 127.6.140.130:47603 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.130:47603): Connection reset by peer (error 104) 04:44:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.291834 6706 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 7165 04:44:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.294353 6722 connection.cc:512] server connection from 127.6.140.131:46125 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.131:46125): Connection reset by peer (error 104) 04:44:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.294351 6848 connection.cc:512] server connection from 127.6.140.131:58318 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.131:58318): Connection reset by peer (error 104) 04:44:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.294359 6706 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 6713 04:44:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.294531 6785 connection.cc:512] server connection from 127.6.140.131:40855 recv error: Network error: failed to read from TLS socket (remote: 127.6.140.131:40855): Connection reset by peer (error 104) 04:44:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.296929 6706 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 6775 04:44:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.299027 6706 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 6836 04:44:10.306 [ERROR - Test worker] (RetryRule.java:76) testKillLeaderBeforeCreateTable(org.apache.kudu.client.TestMasterFailover): failed run 1 org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.140.190:58136,127.6.140.189:38963,127.6.140.188:58759) has no leader. at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:279) at org.apache.kudu.client.ConnectToCluster.access$100(ConnectToCluster.java:47) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:323) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:312) at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:247) at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:294) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:269) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.messageReceived(Connection.java:391) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 04:44:10.307 [INFO - Test worker] (BaseKuduTest.java:77) Creating a new MiniKuduCluster... 04:44:10.307 [INFO - Test worker] (MiniKuduCluster.java:144) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 04:44:10.307 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 04:44:10.307 [INFO - Test worker] (MiniKuduCluster.java:195) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 04:44:10.308 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164" hms_mode: NONE mini_kdc_options { } } 04:44:10.320 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:10.320 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: start_cluster { } 04:44:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.324532 7314 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/wal 04:44:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data 04:44:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.190:38478 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.190 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/logs 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data/info.pb 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 with env {} 04:44:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.335271 7321 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.335372 7321 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.336211 7321 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.336227 7321 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.336233 7321 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.336238 7321 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.337293 7321 master_main.cc:73] Master server non-default flags: 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/wal 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.190:38478 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data/info.pb 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.190 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.7321 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/logs 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.339242 7321 master_main.cc:80] Initializing master server... 04:44:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.339390 7321 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.339485 7321 fs_manager.cc:262] Metadata directory not provided 04:44:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.339507 7321 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.339562 7321 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.339568 7321 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.340225 7321 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data/instance: 04:44:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "dd43f9d452304be18f1b983253c48cae" 04:44:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.340451 7321 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/wal/instance: 04:44:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "dd43f9d452304be18f1b983253c48cae" 04:44:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.342383 7321 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.342535 7321 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.342598 7321 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.343470 7321 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.343497 7321 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/wal 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "dd43f9d452304be18f1b983253c48cae" 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.343541 7321 fs_report.cc:352] FS layout report 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/wal 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/wal 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data/data 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.372488 7321 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.372684 7321 master_main.cc:83] Starting Master server... 04:44:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376010 7321 rpc_server.cc:205] RPC server started. Bound to: 127.7.36.190:38478 04:44:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376072 7321 webserver.cc:175] Starting webserver on 127.7.36.190:0 04:44:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376080 7321 webserver.cc:186] Document root disabled 04:44:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376253 7321 webserver.cc:313] Webserver started. Bound to: http://127.7.36.190:41639/ 04:44:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376382 7321 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-0/data/info.pb 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376670 7380 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.376817 7314 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 7321 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.377048 7314 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/wal 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.189:60666 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.189 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/logs 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data/info.pb 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 with env {} 04:44:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.378743 7380 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.381480 7380 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.381690 7380 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.36.189:60666: Network error: Client connection negotiation failed: client connection to 127.7.36.189:60666: connect: Connection refused (error 111) 04:44:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.387308 7381 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.387398 7381 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.388155 7381 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.388175 7381 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.388180 7381 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.388185 7381 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.389236 7381 master_main.cc:73] Master server non-default flags: 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/wal 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.189:60666 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data/info.pb 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.189 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.7381 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/logs 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.391180 7381 master_main.cc:80] Initializing master server... 04:44:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.391331 7381 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.391422 7381 fs_manager.cc:262] Metadata directory not provided 04:44:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.391444 7381 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.391496 7381 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.391502 7381 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.392130 7381 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data/instance: 04:44:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" 04:44:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.392351 7381 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/wal/instance: 04:44:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" 04:44:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.394311 7381 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.394461 7381 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.394526 7381 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.395398 7381 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.395426 7381 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/wal 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.395468 7381 fs_report.cc:352] FS layout report 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/wal 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/wal 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data/data 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.400851 7380 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } attempt: 1 04:44:10.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.401067 7380 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.36.189:60666: Network error: Client connection negotiation failed: client connection to 127.7.36.189:60666: connect: Connection refused (error 111) 04:44:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.409763 7381 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.409943 7381 master_main.cc:83] Starting Master server... 04:44:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.412647 7381 rpc_server.cc:205] RPC server started. Bound to: 127.7.36.189:60666 04:44:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.412705 7381 webserver.cc:175] Starting webserver on 127.7.36.189:0 04:44:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.412712 7381 webserver.cc:186] Document root disabled 04:44:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.412856 7381 webserver.cc:313] Webserver started. Bound to: http://127.7.36.189:58703/ 04:44:10.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.412971 7381 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-1/data/info.pb 04:44:10.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.413228 7442 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:10.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.414602 7442 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.416497 7442 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.418411 7442 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.418560 7314 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 7381 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.418601 7442 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.36.188:50484: Network error: Client connection negotiation failed: client connection to 127.7.36.188:50484: connect: Connection refused (error 111) 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.418850 7314 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/wal 04:44:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.188:50484 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.188 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/logs 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data/info.pb 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 with env {} 04:44:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.431334 7445 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.431432 7445 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.432399 7445 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.432415 7445 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.432420 7445 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.432426 7445 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.433665 7445 master_main.cc:73] Master server non-default flags: 04:44:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data 04:44:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/wal 04:44:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.188:50484 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data/info.pb 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.188 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.7445 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/logs 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.436373 7445 master_main.cc:80] Initializing master server... 04:44:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.436591 7445 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.436748 7445 fs_manager.cc:262] Metadata directory not provided 04:44:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.436846 7445 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.436987 7445 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.437002 7445 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.437678 7445 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data/instance: 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "6a60472a479c447481e1d06a50e5c491" 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.437779 7442 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } attempt: 1 04:44:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.437913 7445 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/wal/instance: 04:44:10.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "6a60472a479c447481e1d06a50e5c491" 04:44:10.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.439369 7442 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.36.188:50484: Network error: Client connection negotiation failed: client connection to 127.7.36.188:50484: connect: Connection refused (error 111) 04:44:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.440037 7445 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.002s 04:44:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.440312 7445 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.440378 7445 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.441340 7445 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.441367 7445 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/wal 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "6a60472a479c447481e1d06a50e5c491" 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.441416 7445 fs_report.cc:352] FS layout report 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/wal 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/wal 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data/data 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.449249 7380 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } attempt: 2 04:44:10.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.451284 7380 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.451531 7380 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.36.188:50484: Network error: Client connection negotiation failed: client connection to 127.7.36.188:50484: connect: Connection refused (error 111) 04:44:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.459298 7445 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.459554 7445 master_main.cc:83] Starting Master server... 04:44:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.462478 7445 rpc_server.cc:205] RPC server started. Bound to: 127.7.36.188:50484 04:44:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.462536 7445 webserver.cc:175] Starting webserver on 127.7.36.188:0 04:44:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.462543 7445 webserver.cc:186] Document root disabled 04:44:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.462901 7445 webserver.cc:313] Webserver started. Bound to: http://127.7.36.188:46980/ 04:44:10.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.463135 7445 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/master-2/data/info.pb 04:44:10.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.463352 7504 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.464681 7504 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.466614 7504 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.467964 7504 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.470002 7504 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491: Bootstrap starting. 04:44:10.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.470245 7504 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491: No blocks or log segments found. Creating new log. 04:44:10.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.470369 7504 log.cc:525] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491: Log is configured to *not* fsync() on all Append() calls 04:44:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.471019 7504 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491: No bootstrap required, opened a new log 04:44:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.471891 7504 raft_consensus.cc:325] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } 04:44:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.471909 7504 raft_consensus.cc:351] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.471916 7504 raft_consensus.cc:630] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a60472a479c447481e1d06a50e5c491, State: Initialized, Role: FOLLOWER 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.471968 7504 consensus_queue.cc:228] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.472071 7504 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } } 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.472081 7504 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.472088 7508 sys_catalog.cc:337] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } } 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.472096 7508 sys_catalog.cc:340] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.472127 7504 sys_catalog.cc:424] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491 [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.472416 7445 master_main.cc:86] Master server successfully started. 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.472561 7509 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.472579 7509 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.473340 7314 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 7445 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.473603 7314 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/wal 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.129:0 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.36.129 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.129 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/logs 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data/info.pb 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.483784 7511 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.483883 7511 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.484781 7511 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.484813 7511 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.36.129 04:44:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.485873 7511 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/wal 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.129:0 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data/info.pb 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.129 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.7511 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.36.129 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/logs 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.487571 7442 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } attempt: 2 04:44:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.488883 7511 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.489130 7511 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.489229 7511 fs_manager.cc:262] Metadata directory not provided 04:44:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.489246 7511 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.489300 7511 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.489307 7511 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.490087 7511 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data/instance: 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9deb1b243ee34b308e4567a00f6a7ad5" 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.490483 7511 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/wal/instance: 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9deb1b243ee34b308e4567a00f6a7ad5" 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.490483 7442 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d: Bootstrap starting. 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.490754 7442 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d: No blocks or log segments found. Creating new log. 04:44:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.490908 7442 log.cc:525] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d: Log is configured to *not* fsync() on all Append() calls 04:44:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.491580 7442 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d: No bootstrap required, opened a new log 04:44:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492723 7442 raft_consensus.cc:325] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } 04:44:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492743 7442 raft_consensus.cc:351] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492753 7442 raft_consensus.cc:630] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 17d8d81baa044e2fb41c9df0c1a4ad8d, State: Initialized, Role: FOLLOWER 04:44:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492807 7442 consensus_queue.cc:228] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } 04:44:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492928 7442 sys_catalog.cc:337] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } } 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492939 7442 sys_catalog.cc:340] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492965 7516 sys_catalog.cc:337] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } } 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492980 7516 sys_catalog.cc:340] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.492988 7442 sys_catalog.cc:424] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.493372 7517 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.493700 7517 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.493525 7381 master_main.cc:86] Master server successfully started. 04:44:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.493968 7511 fs_manager.cc:502] Time spent creating directory manager: real 0.003s user 0.001s sys 0.001s 04:44:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.494158 7511 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.494222 7511 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.495093 7511 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.495121 7511 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/wal 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "9deb1b243ee34b308e4567a00f6a7ad5" 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.495157 7511 fs_report.cc:352] FS layout report 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/wal 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/wal 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data/data 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.511777 7511 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.511996 7511 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.512042 7511 tablet_server_main.cc:90] Starting tablet server... 04:44:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.514716 7380 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } attempt: 1 04:44:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517243 7511 rpc_server.cc:205] RPC server started. Bound to: 127.7.36.129:59691 04:44:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517371 7380 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae: Bootstrap starting. 04:44:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517381 7511 webserver.cc:175] Starting webserver on 127.7.36.129:0 04:44:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517388 7511 webserver.cc:186] Document root disabled 04:44:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517562 7511 webserver.cc:313] Webserver started. Bound to: http://127.7.36.129:52657/ 04:44:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517635 7380 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae: No blocks or log segments found. Creating new log. 04:44:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517663 7511 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-0/data/info.pb 04:44:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.517786 7380 log.cc:525] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae: Log is configured to *not* fsync() on all Append() calls 04:44:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.518319 7511 tablet_server_main.cc:93] Tablet server successfully started. 04:44:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.519183 7380 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae: No bootstrap required, opened a new log 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520506 7380 raft_consensus.cc:325] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520529 7380 raft_consensus.cc:351] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520539 7380 raft_consensus.cc:630] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd43f9d452304be18f1b983253c48cae, State: Initialized, Role: FOLLOWER 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520598 7380 consensus_queue.cc:228] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520676 7380 sys_catalog.cc:337] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } } 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520686 7380 sys_catalog.cc:340] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520735 7380 sys_catalog.cc:424] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.521059 7321 master_main.cc:86] Master server successfully started. 04:44:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.521201 7642 catalog_manager.cc:1084] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.521212 7642 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.520953 7641 sys_catalog.cc:337] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dd43f9d452304be18f1b983253c48cae" member_type: VOTER last_known_addr { host: "127.7.36.190" port: 38478 } } peers { permanent_uuid: "17d8d81baa044e2fb41c9df0c1a4ad8d" member_type: VOTER last_known_addr { host: "127.7.36.189" port: 60666 } } peers { permanent_uuid: "6a60472a479c447481e1d06a50e5c491" member_type: VOTER last_known_addr { host: "127.7.36.188" port: 50484 } } } 04:44:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.521298 7641 sys_catalog.cc:340] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.525007 7635 heartbeater.cc:346] Connected to a master server at 127.7.36.189:60666 04:44:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.525024 7635 heartbeater.cc:426] Registering TS with master... 04:44:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.525070 7635 heartbeater.cc:475] Master 127.7.36.189:60666 requested a full tablet report, sending... 04:44:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.525712 7409 ts_manager.cc:98] Registered new tserver with Master: 9deb1b243ee34b308e4567a00f6a7ad5 (127.7.36.129:59691) 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.526412 7314 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 7511 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.526690 7314 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/wal 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.130:0 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.36.130 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.130 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/logs 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data/info.pb 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.527998 7636 heartbeater.cc:346] Connected to a master server at 127.7.36.188:50484 04:44:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.528019 7636 heartbeater.cc:426] Registering TS with master... 04:44:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.528050 7636 heartbeater.cc:475] Master 127.7.36.188:50484 requested a full tablet report, sending... 04:44:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.528347 7471 ts_manager.cc:98] Registered new tserver with Master: 9deb1b243ee34b308e4567a00f6a7ad5 (127.7.36.129:59691) 04:44:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.539582 7634 heartbeater.cc:346] Connected to a master server at 127.7.36.190:38478 04:44:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.539609 7634 heartbeater.cc:426] Registering TS with master... 04:44:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.539645 7634 heartbeater.cc:475] Master 127.7.36.190:38478 requested a full tablet report, sending... 04:44:10.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.540340 7347 ts_manager.cc:98] Registered new tserver with Master: 9deb1b243ee34b308e4567a00f6a7ad5 (127.7.36.129:59691) 04:44:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.542851 7644 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.543155 7644 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.544234 7644 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.544353 7644 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.36.130 04:44:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.545686 7644 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data 04:44:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/wal 04:44:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.130:0 04:44:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data/info.pb 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.130 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.7644 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.36.130 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/logs 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.549463 7644 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.549804 7644 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.549965 7644 fs_manager.cc:262] Metadata directory not provided 04:44:10.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.550071 7644 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.550240 7644 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.550335 7644 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.551249 7644 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data/instance: 04:44:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "19547e8d88d04633b05a041e12a72a0f" 04:44:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.551671 7644 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/wal/instance: 04:44:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "19547e8d88d04633b05a041e12a72a0f" 04:44:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.554318 7644 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.554477 7644 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.554533 7644 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.556017 7644 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.556040 7644 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/wal 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "19547e8d88d04633b05a041e12a72a0f" 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.556075 7644 fs_report.cc:352] FS layout report 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/wal 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/wal 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data/data 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.581924 7644 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.582479 7644 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.582679 7644 tablet_server_main.cc:90] Starting tablet server... 04:44:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.585821 7644 rpc_server.cc:205] RPC server started. Bound to: 127.7.36.130:36185 04:44:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.586076 7644 webserver.cc:175] Starting webserver on 127.7.36.130:0 04:44:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.586271 7644 webserver.cc:186] Document root disabled 04:44:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.586583 7644 webserver.cc:313] Webserver started. Bound to: http://127.7.36.130:59394/ 04:44:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.586854 7644 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-1/data/info.pb 04:44:10.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.590083 7314 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 7644 04:44:10.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.590400 7314 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/wal 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.131:0 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.36.131 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.131 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/logs 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data/info.pb 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.591970 7644 tablet_server_main.cc:93] Tablet server successfully started. 04:44:10.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.593730 7764 heartbeater.cc:346] Connected to a master server at 127.7.36.189:60666 04:44:10.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.593750 7764 heartbeater.cc:426] Registering TS with master... 04:44:10.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.593791 7764 heartbeater.cc:475] Master 127.7.36.189:60666 requested a full tablet report, sending... 04:44:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.594157 7409 ts_manager.cc:98] Registered new tserver with Master: 19547e8d88d04633b05a041e12a72a0f (127.7.36.130:36185) 04:44:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.595846 7765 heartbeater.cc:346] Connected to a master server at 127.7.36.188:50484 04:44:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.595870 7765 heartbeater.cc:426] Registering TS with master... 04:44:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.595901 7765 heartbeater.cc:475] Master 127.7.36.188:50484 requested a full tablet report, sending... 04:44:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.596207 7471 ts_manager.cc:98] Registered new tserver with Master: 19547e8d88d04633b05a041e12a72a0f (127.7.36.130:36185) 04:44:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.596719 7763 heartbeater.cc:346] Connected to a master server at 127.7.36.190:38478 04:44:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.596738 7763 heartbeater.cc:426] Registering TS with master... 04:44:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.596767 7763 heartbeater.cc:475] Master 127.7.36.190:38478 requested a full tablet report, sending... 04:44:10.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.597141 7347 ts_manager.cc:98] Registered new tserver with Master: 19547e8d88d04633b05a041e12a72a0f (127.7.36.130:36185) 04:44:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.609441 7768 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.610021 7768 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.611194 7768 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.611392 7768 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.36.131 04:44:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.612772 7768 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data 04:44:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/wal 04:44:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.36.131:0 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data/info.pb 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.36.131 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.36.190:38478,127.7.36.189:60666,127.7.36.188:50484 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.7768 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.36.131 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/logs 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.616338 7768 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.616762 7768 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.617064 7768 fs_manager.cc:262] Metadata directory not provided 04:44:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.617269 7768 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.617480 7768 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.617645 7768 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.618623 7768 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data/instance: 04:44:10.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "5d53d7aa204246b8aef27b3883746a27" 04:44:10.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.619164 7768 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/wal/instance: 04:44:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "5d53d7aa204246b8aef27b3883746a27" 04:44:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.623003 7336 tablet.cc:1728] T 00000000000000000000000000000000 P dd43f9d452304be18f1b983253c48cae: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.623275 7768 fs_manager.cc:502] Time spent creating directory manager: real 0.004s user 0.001s sys 0.001s 04:44:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.623471 7768 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.623531 7768 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.626152 7768 fs_manager.cc:418] Time spent opening block manager: real 0.002s user 0.000s sys 0.000s 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.626178 7768 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/wal 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "5d53d7aa204246b8aef27b3883746a27" 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.626219 7768 fs_report.cc:352] FS layout report 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/wal 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/wal 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data/data 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.660192 7398 tablet.cc:1728] T 00000000000000000000000000000000 P 17d8d81baa044e2fb41c9df0c1a4ad8d: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.673300 7768 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.673820 7768 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.674019 7768 tablet_server_main.cc:90] Starting tablet server... 04:44:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.679695 7768 rpc_server.cc:205] RPC server started. Bound to: 127.7.36.131:52398 04:44:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.679759 7768 webserver.cc:175] Starting webserver on 127.7.36.131:0 04:44:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.679765 7768 webserver.cc:186] Document root disabled 04:44:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.679944 7768 webserver.cc:313] Webserver started. Bound to: http://127.7.36.131:45047/ 04:44:10.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.680088 7768 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster2261015268264894164/ts-2/data/info.pb 04:44:10.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.680645 7768 tablet_server_main.cc:93] Tablet server successfully started. 04:44:10.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.684717 7889 heartbeater.cc:346] Connected to a master server at 127.7.36.189:60666 04:44:10.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.684737 7889 heartbeater.cc:426] Registering TS with master... 04:44:10.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.684779 7889 heartbeater.cc:475] Master 127.7.36.189:60666 requested a full tablet report, sending... 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685128 7888 heartbeater.cc:346] Connected to a master server at 127.7.36.190:38478 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685142 7888 heartbeater.cc:426] Registering TS with master... 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685191 7888 heartbeater.cc:475] Master 127.7.36.190:38478 requested a full tablet report, sending... 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685250 7409 ts_manager.cc:98] Registered new tserver with Master: 5d53d7aa204246b8aef27b3883746a27 (127.7.36.131:52398) 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685390 7890 heartbeater.cc:346] Connected to a master server at 127.7.36.188:50484 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685403 7890 heartbeater.cc:426] Registering TS with master... 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685436 7890 heartbeater.cc:475] Master 127.7.36.188:50484 requested a full tablet report, sending... 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685448 7347 ts_manager.cc:98] Registered new tserver with Master: 5d53d7aa204246b8aef27b3883746a27 (127.7.36.131:52398) 04:44:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.685724 7471 ts_manager.cc:98] Registered new tserver with Master: 5d53d7aa204246b8aef27b3883746a27 (127.7.36.131:52398) 04:44:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.688235 7314 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 7768 04:44:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.694262 7314 external_mini_cluster.cc:513] 3 TS(s) registered with all masters 04:44:10.694 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:10.694 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_masters { } 04:44:10.695 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.7.36.190" port: 38478 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.7.36.189" port: 60666 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.7.36.188" port: 50484 } } } 04:44:10.695 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_tservers { } 04:44:10.696 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.7.36.129" port: 59691 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.7.36.130" port: 36185 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.7.36.131" port: 52398 } } } 04:44:10.696 [INFO - Test worker] (BaseKuduTest.java:81) Creating a new Kudu client... 04:44:10.702 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:10.707 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.709866 7460 tablet.cc:1728] T 00000000000000000000000000000000 P 6a60472a479c447481e1d06a50e5c491: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:10.710 [DEBUG - New I/O worker #82] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.36.189 04:44:10.711 [DEBUG - New I/O worker #84] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.36.190 04:44:10.712 [DEBUG - New I/O worker #83] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.36.188 04:44:10.725 [DEBUG - New I/O worker #84] (Negotiator.java:764) Authenticated connection [id: 0x586d6752, /127.7.36.190:54735 => /127.7.36.190:38478] using SASL/PLAIN 04:44:10.725 [DEBUG - New I/O worker #82] (Negotiator.java:764) Authenticated connection [id: 0x58d4299c, /127.7.36.189:50549 => /127.7.36.189:60666] using SASL/PLAIN 04:44:10.725 [DEBUG - New I/O worker #83] (Negotiator.java:764) Authenticated connection [id: 0x1cf9b53e, /127.7.36.188:48153 => /127.7.36.188:50484] using SASL/PLAIN 04:44:10.726 [WARN - New I/O worker #84] (ConnectToCluster.java:278) None of the provided masters 127.7.36.189:60666,127.7.36.188:50484,127.7.36.190:38478 is a leader; will retry 04:44:10.726 [DEBUG - Time-limited test] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:10.732 [DEBUG - New I/O worker #82] (Connection.java:688) [peer master-127.7.36.189:60666(127.7.36.189:60666)] cleaning up while in state READY due to: connection disconnected 04:44:10.732 [DEBUG - New I/O worker #83] (Connection.java:688) [peer master-127.7.36.188:50484(127.7.36.188:50484)] cleaning up while in state READY due to: connection disconnected 04:44:10.733 [DEBUG - New I/O worker #84] (Connection.java:688) [peer master-127.7.36.190:38478(127.7.36.190:38478)] cleaning up while in state READY due to: connection disconnected 04:44:10.733 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:10.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.734905 7314 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 7511 04:44:10.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.737833 7390 connection.cc:512] server connection from 127.7.36.129:47013 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.129:47013): Connection reset by peer (error 104) 04:44:10.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.737841 7455 connection.cc:512] server connection from 127.7.36.129:59886 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.129:59886): Connection reset by peer (error 104) 04:44:10.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.737860 7328 connection.cc:512] server connection from 127.7.36.129:53663 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.129:53663): Connection reset by peer (error 104) 04:44:10.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.738265 7314 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 7644 04:44:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.740942 7391 connection.cc:512] server connection from 127.7.36.130:48221 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.130:48221): Connection reset by peer (error 104) 04:44:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.740952 7452 connection.cc:512] server connection from 127.7.36.130:46511 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.130:46511): Connection reset by peer (error 104) 04:44:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.740973 7329 connection.cc:512] server connection from 127.7.36.130:41805 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.130:41805): Connection reset by peer (error 104) 04:44:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.741063 7314 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 7768 04:44:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.743419 7393 connection.cc:512] server connection from 127.7.36.131:45140 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.131:45140): Connection reset by peer (error 104) 04:44:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.743428 7330 connection.cc:512] server connection from 127.7.36.131:39245 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.131:39245): Connection reset by peer (error 104) 04:44:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.743438 7454 connection.cc:512] server connection from 127.7.36.131:46039 recv error: Network error: failed to read from TLS socket (remote: 127.7.36.131:46039): Connection reset by peer (error 104) 04:44:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.743520 7314 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 7321 04:44:10.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.745971 7314 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 7381 04:44:10.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.748116 7314 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 7445 04:44:10.755 [ERROR - Test worker] (RetryRule.java:76) testKillLeaderBeforeCreateTable(org.apache.kudu.client.TestMasterFailover): failed run 2 org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.36.189:60666,127.7.36.188:50484,127.7.36.190:38478) has no leader. at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:279) at org.apache.kudu.client.ConnectToCluster.access$100(ConnectToCluster.java:47) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:323) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:312) at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:247) at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:294) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:269) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.messageReceived(Connection.java:391) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 04:44:10.755 [INFO - Test worker] (BaseKuduTest.java:77) Creating a new MiniKuduCluster... 04:44:10.755 [INFO - Test worker] (MiniKuduCluster.java:144) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 04:44:10.756 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 04:44:10.756 [INFO - Test worker] (MiniKuduCluster.java:195) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 04:44:10.757 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263" hms_mode: NONE mini_kdc_options { } } 04:44:10.768 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:10.768 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: start_cluster { } 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.772470 7918 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/wal 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.190:57785 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.190 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/logs 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data/info.pb 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 with env {} 04:44:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.782817 7925 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.782910 7925 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.783715 7925 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.783731 7925 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.783735 7925 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.783738 7925 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.784775 7925 master_main.cc:73] Master server non-default flags: 04:44:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data 04:44:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/wal 04:44:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.190:57785 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data/info.pb 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.190 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.7925 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/logs 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.786501 7925 master_main.cc:80] Initializing master server... 04:44:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.786626 7925 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.786705 7925 fs_manager.cc:262] Metadata directory not provided 04:44:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.786716 7925 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.786754 7925 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.786757 7925 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.787519 7925 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data/instance: 04:44:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "1437f6a196844200834dbf1e14faeb3e" 04:44:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.787748 7925 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/wal/instance: 04:44:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "1437f6a196844200834dbf1e14faeb3e" 04:44:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.791874 7925 fs_manager.cc:502] Time spent creating directory manager: real 0.004s user 0.000s sys 0.002s 04:44:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.792049 7925 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.792109 7925 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.793330 7925 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.793361 7925 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/wal 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "1437f6a196844200834dbf1e14faeb3e" 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.793406 7925 fs_report.cc:352] FS layout report 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/wal 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/wal 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data/data 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.820163 7925 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.820327 7925 master_main.cc:83] Starting Master server... 04:44:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.823420 7925 rpc_server.cc:205] RPC server started. Bound to: 127.7.187.190:57785 04:44:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.823482 7925 webserver.cc:175] Starting webserver on 127.7.187.190:0 04:44:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.823487 7925 webserver.cc:186] Document root disabled 04:44:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.823626 7925 webserver.cc:313] Webserver started. Bound to: http://127.7.187.190:37665/ 04:44:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.823748 7925 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-0/data/info.pb 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.824008 7984 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.824291 7918 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 7925 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.824645 7918 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/wal 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.189:45658 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.189 04:44:10.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/logs 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data/info.pb 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 with env {} 04:44:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.826936 7984 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.837913 7984 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.838428 7984 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.187.189:45658: Network error: Client connection negotiation failed: client connection to 127.7.187.189:45658: connect: Connection refused (error 111) 04:44:10.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.841804 7985 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.841928 7985 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.842978 7985 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.842994 7985 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.843000 7985 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.843005 7985 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.844339 7985 master_main.cc:73] Master server non-default flags: 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/wal 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.189:45658 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data/info.pb 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.189 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.7985 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/logs 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.846633 7985 master_main.cc:80] Initializing master server... 04:44:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.846789 7985 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.846873 7985 fs_manager.cc:262] Metadata directory not provided 04:44:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.846885 7985 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.846933 7985 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.846938 7985 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.847731 7985 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data/instance: 04:44:10.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "59d75574fac2412495f09e5edff872fe" 04:44:10.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.847949 7985 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/wal/instance: 04:44:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "59d75574fac2412495f09e5edff872fe" 04:44:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.850499 7985 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.850656 7985 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.850710 7985 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.851872 7985 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:10.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.851899 7985 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/wal 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "59d75574fac2412495f09e5edff872fe" 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.851940 7985 fs_report.cc:352] FS layout report 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/wal 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/wal 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data/data 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.857609 7984 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } attempt: 1 04:44:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.857939 7984 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.187.189:45658: Network error: Client connection negotiation failed: client connection to 127.7.187.189:45658: connect: Connection refused (error 111) 04:44:10.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.876606 7985 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.876772 7985 master_main.cc:83] Starting Master server... 04:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.880178 7985 rpc_server.cc:205] RPC server started. Bound to: 127.7.187.189:45658 04:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.880236 7985 webserver.cc:175] Starting webserver on 127.7.187.189:0 04:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.880244 7985 webserver.cc:186] Document root disabled 04:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.880400 7985 webserver.cc:313] Webserver started. Bound to: http://127.7.187.189:44786/ 04:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.880532 7985 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-1/data/info.pb 04:44:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.880682 7918 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 7985 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.881040 7918 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/wal 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=localhost 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.188:47280 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.188 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/logs 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data/info.pb 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 with env {} 04:44:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.882460 8046 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.884654 8046 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.887917 8046 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.898030 8046 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.898531 8046 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.187.188:47280: Network error: Client connection negotiation failed: client connection to 127.7.187.188:47280: connect: Connection refused (error 111) 04:44:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.899842 8047 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.899971 8047 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.901073 8047 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 04:44:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.901095 8047 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.901101 8047 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 04:44:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.901108 8047 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.902565 8047 master_main.cc:73] Master server non-default flags: 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/wal 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_ca_key_size=1024 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --master_addresses=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tsk_num_rsa_bits=512 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.188:47280 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_reuseport=true 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data/info.pb 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.188 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-master.8047 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/logs 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Master server version: 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.904970 8047 master_main.cc:80] Initializing master server... 04:44:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.905154 8047 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.905246 8047 fs_manager.cc:262] Metadata directory not provided 04:44:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.905269 8047 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.905324 8047 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.905329 8047 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.906107 7984 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } attempt: 2 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.906226 8047 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data/instance: 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "8fab3b9305924cf49601eb715654619e" 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.906733 8047 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/wal/instance: 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "8fab3b9305924cf49601eb715654619e" 04:44:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.907744 7984 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.907953 7984 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.187.188:47280: Network error: Client connection negotiation failed: client connection to 127.7.187.188:47280: connect: Connection refused (error 111) 04:44:10.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.909406 8047 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.001s sys 0.001s 04:44:10.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.909574 8047 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.909639 8047 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.910866 8047 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s 04:44:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.910895 8047 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/wal 04:44:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "8fab3b9305924cf49601eb715654619e" 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.910938 8047 fs_report.cc:352] FS layout report 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/wal 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/wal 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data/data 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.917697 8046 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } attempt: 1 04:44:10.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.917981 8046 consensus_peers.cc:565] Error getting permanent uuid from config peer 127.7.187.188:47280: Network error: Client connection negotiation failed: client connection to 127.7.187.188:47280: connect: Connection refused (error 111) 04:44:10.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.937855 8047 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.938187 8047 master_main.cc:83] Starting Master server... 04:44:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.941334 8047 rpc_server.cc:205] RPC server started. Bound to: 127.7.187.188:47280 04:44:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.941404 8047 webserver.cc:175] Starting webserver on 127.7.187.188:0 04:44:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.941411 8047 webserver.cc:186] Document root disabled 04:44:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.941603 8047 webserver.cc:313] Webserver started. Bound to: http://127.7.187.188:49369/ 04:44:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.941856 8047 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/master-2/data/info.pb 04:44:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.942155 8108 data_dirs.cc:934] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 04:44:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.943882 8108 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.946108 8108 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.946785 7918 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master as pid 8047 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.947221 7918 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/wal 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.129:0 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.187.129 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.129 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/logs 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data/info.pb 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:10.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.948300 8108 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } has no permanent_uuid. Determining permanent_uuid... 04:44:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.952353 8108 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e: Bootstrap starting. 04:44:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.952641 8108 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e: No blocks or log segments found. Creating new log. 04:44:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.952864 8108 log.cc:525] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e: Log is configured to *not* fsync() on all Append() calls 04:44:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.953425 8108 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e: No bootstrap required, opened a new log 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954422 8108 raft_consensus.cc:325] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954442 8108 raft_consensus.cc:351] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954452 8108 raft_consensus.cc:630] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8fab3b9305924cf49601eb715654619e, State: Initialized, Role: FOLLOWER 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954514 8108 consensus_queue.cc:228] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954607 8108 sys_catalog.cc:337] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } } 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954617 8108 sys_catalog.cc:340] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954660 8113 sys_catalog.cc:337] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } } 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954674 8113 sys_catalog.cc:340] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.954685 8108 sys_catalog.cc:424] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.954998 8114 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.955011 8114 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.960340 8047 master_main.cc:86] Master server successfully started. 04:44:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.961062 8110 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.961239 8110 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.962276 8110 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.962313 8110 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.187.129 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.963652 8110 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/wal 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.129:0 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data/info.pb 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.129 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.8110 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.187.129 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/logs 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966161 8046 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } attempt: 2 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966192 8110 tablet_server_main.cc:85] Initializing tablet server... 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966574 8110 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572194us 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966666 8110 fs_manager.cc:262] Metadata directory not provided 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966682 8110 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966737 8110 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.966744 8110 server_base.cc:433] Attempting to create new FS layout instead 04:44:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.967728 8110 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data/instance: 04:44:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "ff07607f9e28478f872764f196769705" 04:44:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.967880 8110 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/wal/instance: 04:44:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "ff07607f9e28478f872764f196769705" 04:44:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.968832 8046 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe: Bootstrap starting. 04:44:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.969082 8046 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe: No blocks or log segments found. Creating new log. 04:44:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.969252 8046 log.cc:525] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe: Log is configured to *not* fsync() on all Append() calls 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.970121 8046 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe: No bootstrap required, opened a new log 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.970484 8110 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.002s sys 0.000s 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.970651 8110 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.970721 8110 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971144 7984 consensus_peers.cc:575] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } attempt: 1 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971444 8046 raft_consensus.cc:325] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971458 8046 raft_consensus.cc:351] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971465 8046 raft_consensus.cc:630] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59d75574fac2412495f09e5edff872fe, State: Initialized, Role: FOLLOWER 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971531 8046 consensus_queue.cc:228] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971657 8046 sys_catalog.cc:337] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } } 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971665 8046 sys_catalog.cc:340] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971693 8122 sys_catalog.cc:337] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } } 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971706 8122 sys_catalog.cc:340] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.971715 8046 sys_catalog.cc:424] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.972074 7985 master_main.cc:86] Master server successfully started. 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.972162 8123 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.972172 8123 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.972415 8110 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.972445 8110 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/wal 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "ff07607f9e28478f872764f196769705" 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:10 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.972492 8110 fs_report.cc:352] FS layout report 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/wal 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/wal 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data/data 04:44:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.981762 7984 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e: Bootstrap starting. 04:44:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.982033 7984 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e: No blocks or log segments found. Creating new log. 04:44:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.982203 7984 log.cc:525] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e: Log is configured to *not* fsync() on all Append() calls 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.991719 7984 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e: No bootstrap required, opened a new log 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993155 7984 raft_consensus.cc:325] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993170 7984 raft_consensus.cc:351] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993177 7984 raft_consensus.cc:630] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1437f6a196844200834dbf1e14faeb3e, State: Initialized, Role: FOLLOWER 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993239 7984 consensus_queue.cc:228] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993341 7984 sys_catalog.cc:337] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } } 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993351 7984 sys_catalog.cc:340] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993402 7984 sys_catalog.cc:424] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [sys.catalog]: configured and running, proceeding with master startup. 04:44:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.993548 7925 master_main.cc:86] Master server successfully started. 04:44:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.995692 8110 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.995925 8110 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.995977 8110 tablet_server_main.cc:90] Starting tablet server... 04:44:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.997221 8131 sys_catalog.cc:337] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1437f6a196844200834dbf1e14faeb3e" member_type: VOTER last_known_addr { host: "127.7.187.190" port: 57785 } } peers { permanent_uuid: "59d75574fac2412495f09e5edff872fe" member_type: VOTER last_known_addr { host: "127.7.187.189" port: 45658 } } peers { permanent_uuid: "8fab3b9305924cf49601eb715654619e" member_type: VOTER last_known_addr { host: "127.7.187.188" port: 47280 } } } 04:44:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.997488 8131 sys_catalog.cc:340] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e [sys.catalog]: This master's current role is: FOLLOWER 04:44:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.997896 8132 catalog_manager.cc:1084] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e: acquiring CA information for follower catalog manager: Not found: root CA entry not found 04:44:10.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:10.998071 8132 catalog_manager.cc:583] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 04:44:10.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.998929 8110 rpc_server.cc:205] RPC server started. Bound to: 127.7.187.129:35966 04:44:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.999071 8110 webserver.cc:175] Starting webserver on 127.7.187.129:0 04:44:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.999183 8110 webserver.cc:186] Document root disabled 04:44:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.999433 8110 webserver.cc:313] Webserver started. Bound to: http://127.7.187.129:46882/ 04:44:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:10.999656 8110 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-0/data/info.pb 04:44:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.000515 8110 tablet_server_main.cc:93] Tablet server successfully started. 04:44:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.003513 8244 heartbeater.cc:346] Connected to a master server at 127.7.187.188:47280 04:44:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.003532 8244 heartbeater.cc:426] Registering TS with master... 04:44:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.003574 8244 heartbeater.cc:475] Master 127.7.187.188:47280 requested a full tablet report, sending... 04:44:11.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.004025 8075 ts_manager.cc:98] Registered new tserver with Master: ff07607f9e28478f872764f196769705 (127.7.187.129:35966) 04:44:11.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.009599 7918 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 8110 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.009896 7918 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/wal 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.130:0 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.187.130 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.130 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/logs 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data/info.pb 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.010123 8242 heartbeater.cc:346] Connected to a master server at 127.7.187.190:57785 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.010135 8242 heartbeater.cc:426] Registering TS with master... 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.010166 8242 heartbeater.cc:475] Master 127.7.187.190:57785 requested a full tablet report, sending... 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.010470 7951 ts_manager.cc:98] Registered new tserver with Master: ff07607f9e28478f872764f196769705 (127.7.187.129:35966) 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.010855 8243 heartbeater.cc:346] Connected to a master server at 127.7.187.189:45658 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.010938 8243 heartbeater.cc:426] Registering TS with master... 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.011096 8243 heartbeater.cc:475] Master 127.7.187.189:45658 requested a full tablet report, sending... 04:44:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.011445 8013 ts_manager.cc:98] Registered new tserver with Master: ff07607f9e28478f872764f196769705 (127.7.187.129:35966) 04:44:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.020998 8248 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.021095 8248 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:11.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.021951 8248 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:11.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.021981 8248 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.187.130 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.023008 8248 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/wal 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.130:0 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data/info.pb 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.130 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.8248 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.187.130 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/logs 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.025063 8248 tablet_server_main.cc:85] Initializing tablet server... 04:44:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.025384 8248 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572694us 04:44:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.025460 8248 fs_manager.cc:262] Metadata directory not provided 04:44:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.025475 8248 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.025527 8248 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.025534 8248 server_base.cc:433] Attempting to create new FS layout instead 04:44:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.026288 8248 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data/instance: 04:44:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a97c6ad1dba1438fa6a67b336c7fa130" 04:44:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:11 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.026593 8248 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/wal/instance: 04:44:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a97c6ad1dba1438fa6a67b336c7fa130" 04:44:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:11 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:11.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.042394 8248 fs_manager.cc:502] Time spent creating directory manager: real 0.016s user 0.001s sys 0.001s 04:44:11.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.042593 8248 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:11.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.042661 8248 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.049319 8248 fs_manager.cc:418] Time spent opening block manager: real 0.006s user 0.000s sys 0.000s 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.049347 8248 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/wal 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "a97c6ad1dba1438fa6a67b336c7fa130" 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:11 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.049388 8248 fs_report.cc:352] FS layout report 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/wal 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/wal 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data/data 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:11.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.074018 7940 tablet.cc:1728] T 00000000000000000000000000000000 P 1437f6a196844200834dbf1e14faeb3e: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:11.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.130237 8248 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.130461 8248 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.130504 8248 tablet_server_main.cc:90] Starting tablet server... 04:44:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.131836 8002 tablet.cc:1728] T 00000000000000000000000000000000 P 59d75574fac2412495f09e5edff872fe: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.133728 8248 rpc_server.cc:205] RPC server started. Bound to: 127.7.187.130:41050 04:44:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.133785 8248 webserver.cc:175] Starting webserver on 127.7.187.130:0 04:44:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.133792 8248 webserver.cc:186] Document root disabled 04:44:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.133956 8248 webserver.cc:313] Webserver started. Bound to: http://127.7.187.130:49825/ 04:44:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.134081 8248 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-1/data/info.pb 04:44:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.134830 8248 tablet_server_main.cc:93] Tablet server successfully started. 04:44:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.135960 7918 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 8248 04:44:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.136287 7918 external_mini_cluster.cc:817] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver 04:44:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/wal 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --block_manager=log 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.131:0 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.187.131 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.131 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/logs 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data/info.pb 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags 04:44:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags with env {} 04:44:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.139809 8369 heartbeater.cc:346] Connected to a master server at 127.7.187.188:47280 04:44:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.139824 8369 heartbeater.cc:426] Registering TS with master... 04:44:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.139866 8369 heartbeater.cc:475] Master 127.7.187.188:47280 requested a full tablet report, sending... 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.140611 8367 heartbeater.cc:346] Connected to a master server at 127.7.187.190:57785 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.140625 8367 heartbeater.cc:426] Registering TS with master... 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.140652 8367 heartbeater.cc:475] Master 127.7.187.190:57785 requested a full tablet report, sending... 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.140893 7951 ts_manager.cc:98] Registered new tserver with Master: a97c6ad1dba1438fa6a67b336c7fa130 (127.7.187.130:41050) 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.140921 8075 ts_manager.cc:98] Registered new tserver with Master: a97c6ad1dba1438fa6a67b336c7fa130 (127.7.187.130:41050) 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.141680 8368 heartbeater.cc:346] Connected to a master server at 127.7.187.189:45658 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.141696 8368 heartbeater.cc:426] Registering TS with master... 04:44:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.141727 8368 heartbeater.cc:475] Master 127.7.187.189:45658 requested a full tablet report, sending... 04:44:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.142113 8013 ts_manager.cc:98] Registered new tserver with Master: a97c6ad1dba1438fa6a67b336c7fa130 (127.7.187.130:41050) 04:44:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) WARNING: Logging before InitGoogleLogging() is written to STDERR 04:44:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.152184 8372 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 04:44:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.152307 8372 flags.cc:406] Enabled unsafe flag: --never_fsync=true 04:44:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.153300 8372 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 04:44:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.153334 8372 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.187.131 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.154578 8372 tablet_server_main.cc:78] Tablet server non-default flags: 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/wal 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --ipki_server_key_size=1024 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_bind_addresses=127.7.187.131:0 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --rpc_server_allow_ephemeral_ports=true 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --metrics_log_interval_ms=1000 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_format=pb 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data/info.pb 04:44:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_interface=127.7.187.131 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --webserver_port=0 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --tserver_master_addrs=127.7.187.190:57785,127.7.187.189:45658,127.7.187.188:47280 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --never_fsync=true 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --heap_profile_path=/tmp/kudu-tserver.8372 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --redact=none 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_experimental_flags=true 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --unlock_unsafe_flags=true 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --enable_minidumps=false 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --local_ip_for_outbound_sockets=127.7.187.131 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/logs 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logbuflevel=-1 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) --logtostderr=true 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Tablet server version: 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) kudu 1.8.0-SNAPSHOT 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) revision 2de096ccbb245d801a1fd179cdcc1fc6c832bedd 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build type RELEASE 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) built by jenkins at 29 Sep 2018 02:34:30 PST on quasar-hscjoi-1.vpc.cloudera.com 04:44:11.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) build id 151819 04:44:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.156261 8372 tablet_server_main.cc:85] Initializing tablet server... 04:44:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.156450 8372 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 572694us 04:44:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.156519 8372 fs_manager.cc:262] Metadata directory not provided 04:44:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.156535 8372 fs_manager.cc:268] Using write-ahead log directory (fs_wal_dir) as metadata directory 04:44:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.156586 8372 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 04:44:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.156594 8372 server_base.cc:433] Attempting to create new FS layout instead 04:44:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.157239 8372 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data/instance: 04:44:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c78985fd24e2440d8654d05a33adbc55" 04:44:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:11 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.157470 8372 fs_manager.cc:601] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/wal/instance: 04:44:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c78985fd24e2440d8654d05a33adbc55" 04:44:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:11 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.159312 8372 fs_manager.cc:502] Time spent creating directory manager: real 0.002s user 0.000s sys 0.002s 04:44:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.159459 8372 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 04:44:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.159523 8372 file_cache.cc:466] Constructed file cache lbm with capacity 13107 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.160394 8372 fs_manager.cc:418] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.160419 8372 fs_manager.cc:435] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/wal 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) uuid: "c78985fd24e2440d8654d05a33adbc55" 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) format_stamp: "Formatted at 2018-09-29 11:44:11 on quasar-hscjoi-1.vpc.cloudera.com" 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.160455 8372 fs_report.cc:352] FS layout report 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) -------------------- 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/wal 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/wal 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data/data 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live blocks: 0 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes: 0 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total live bytes (after alignment): 0 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total number of LBM containers: 0 (0 full) 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for missing blocks 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Did not check for orphaned blocks 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM containers with extra space: 0 (0 repaired) 04:44:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total full LBM container extra space in bytes: 0 (0 repaired) 04:44:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total incomplete LBM containers: 0 (0 repaired) 04:44:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) Total LBM partial records: 0 (0 repaired) 04:44:11.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.181756 8372 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 04:44:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.181999 8372 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 04:44:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.182051 8372 tablet_server_main.cc:90] Starting tablet server... 04:44:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.187861 8372 rpc_server.cc:205] RPC server started. Bound to: 127.7.187.131:56317 04:44:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.187919 8372 webserver.cc:175] Starting webserver on 127.7.187.131:0 04:44:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.187927 8372 webserver.cc:186] Document root disabled 04:44:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.188138 8372 webserver.cc:313] Webserver started. Bound to: http://127.7.187.131:46922/ 04:44:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.188264 8372 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster3828637648489222263/ts-2/data/info.pb 04:44:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.188563 8064 tablet.cc:1728] T 00000000000000000000000000000000 P 8fab3b9305924cf49601eb715654619e: Can't schedule compaction. Clean time has not been advanced past its initial value. 04:44:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.189098 8372 tablet_server_main.cc:93] Tablet server successfully started. 04:44:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.189548 7918 external_mini_cluster.cc:879] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver as pid 8372 04:44:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193234 8495 heartbeater.cc:346] Connected to a master server at 127.7.187.188:47280 04:44:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193251 8495 heartbeater.cc:426] Registering TS with master... 04:44:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193292 8495 heartbeater.cc:475] Master 127.7.187.188:47280 requested a full tablet report, sending... 04:44:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193642 8494 heartbeater.cc:346] Connected to a master server at 127.7.187.189:45658 04:44:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193658 8494 heartbeater.cc:426] Registering TS with master... 04:44:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193692 8494 heartbeater.cc:475] Master 127.7.187.189:45658 requested a full tablet report, sending... 04:44:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193790 8075 ts_manager.cc:98] Registered new tserver with Master: c78985fd24e2440d8654d05a33adbc55 (127.7.187.131:56317) 04:44:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.193946 8013 ts_manager.cc:98] Registered new tserver with Master: c78985fd24e2440d8654d05a33adbc55 (127.7.187.131:56317) 04:44:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.194568 8493 heartbeater.cc:346] Connected to a master server at 127.7.187.190:57785 04:44:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.194588 8493 heartbeater.cc:426] Registering TS with master... 04:44:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.194617 8493 heartbeater.cc:475] Master 127.7.187.190:57785 requested a full tablet report, sending... 04:44:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.194901 7951 ts_manager.cc:98] Registered new tserver with Master: c78985fd24e2440d8654d05a33adbc55 (127.7.187.131:56317) 04:44:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.198515 7918 external_mini_cluster.cc:513] 3 TS(s) registered with all masters 04:44:11.198 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: 04:44:11.199 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_masters { } 04:44:11.199 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.7.187.190" port: 57785 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.7.187.189" port: 45658 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.7.187.188" port: 47280 } } } 04:44:11.200 [DEBUG - Test worker] (MiniKuduCluster.java:163) Request: get_tservers { } 04:44:11.200 [DEBUG - Test worker] (MiniKuduCluster.java:173) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.7.187.129" port: 35966 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.7.187.130" port: 41050 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.7.187.131" port: 56317 } } } 04:44:11.200 [INFO - Test worker] (BaseKuduTest.java:81) Creating a new Kudu client... 04:44:11.205 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:11.210 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 04:44:11.217 [DEBUG - New I/O worker #102] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.187.189 04:44:11.218 [DEBUG - New I/O worker #101] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.187.188 04:44:11.219 [DEBUG - New I/O worker #100] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.187.190 04:44:11.229 [DEBUG - New I/O worker #102] (Negotiator.java:764) Authenticated connection [id: 0xcdc918d0, /127.7.187.189:48955 => /127.7.187.189:45658] using SASL/PLAIN 04:44:11.234 [DEBUG - New I/O worker #101] (Negotiator.java:764) Authenticated connection [id: 0xf2da55df, /127.7.187.188:60778 => /127.7.187.188:47280] using SASL/PLAIN 04:44:11.235 [DEBUG - New I/O worker #100] (Negotiator.java:764) Authenticated connection [id: 0xbb5a7f2c, /127.7.187.190:58585 => /127.7.187.190:57785] using SASL/PLAIN 04:44:11.235 [WARN - New I/O worker #100] (ConnectToCluster.java:278) None of the provided masters 127.7.187.190:57785,127.7.187.188:47280,127.7.187.189:45658 is a leader; will retry 04:44:11.236 [DEBUG - Time-limited test] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:11.238 [DEBUG - New I/O worker #100] (Connection.java:688) [peer master-127.7.187.190:57785(127.7.187.190:57785)] cleaning up while in state READY due to: connection disconnected 04:44:11.239 [DEBUG - New I/O worker #102] (Connection.java:688) [peer master-127.7.187.189:45658(127.7.187.189:45658)] cleaning up while in state READY due to: connection disconnected 04:44:11.239 [DEBUG - Test worker] (AsyncKuduClient.java:2131) Releasing all remaining resources 04:44:11.239 [DEBUG - New I/O worker #101] (Connection.java:688) [peer master-127.7.187.188:47280(127.7.187.188:47280)] cleaning up while in state READY due to: connection disconnected 04:44:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.240550 7918 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 8110 04:44:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.242785 8059 connection.cc:512] server connection from 127.7.187.129:57815 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.129:57815): Connection reset by peer (error 104) 04:44:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.242789 7933 connection.cc:512] server connection from 127.7.187.129:32896 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.129:32896): Connection reset by peer (error 104) 04:44:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.242801 7997 connection.cc:512] server connection from 127.7.187.129:54918 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.129:54918): Connection reset by peer (error 104) 04:44:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.242945 7918 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 8248 04:44:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.245532 8058 connection.cc:512] server connection from 127.7.187.130:41392 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.130:41392): Connection reset by peer (error 104) 04:44:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.245542 7933 connection.cc:512] server connection from 127.7.187.130:36472 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.130:36472): Connection reset by peer (error 104) 04:44:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.245560 7995 connection.cc:512] server connection from 127.7.187.130:42635 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.130:42635): Connection reset by peer (error 104) 04:44:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.245657 7918 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-tserver with pid 8372 04:44:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.247828 8056 connection.cc:512] server connection from 127.7.187.131:60134 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.131:60134): Connection reset by peer (error 104) 04:44:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.247834 7935 connection.cc:512] server connection from 127.7.187.131:36432 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.131:36432): Connection reset by peer (error 104) 04:44:11.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) W0929 04:44:11.247932 7995 connection.cc:512] server connection from 127.7.187.131:44463 recv error: Network error: failed to read from TLS socket (remote: 127.7.187.131:44463): Connection reset by peer (error 104) 04:44:11.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.248046 7918 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 7925 04:44:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.251262 7918 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 7985 04:44:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:533) I0929 04:44:11.254074 7918 external_mini_cluster.cc:1015] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/release/bin/kudu-master with pid 8047 04:44:11.261 [ERROR - Test worker] (RetryRule.java:76) testKillLeaderBeforeCreateTable(org.apache.kudu.client.TestMasterFailover): failed run 3 org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.187.190:57785,127.7.187.188:47280,127.7.187.189:45658) has no leader. at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:279) at org.apache.kudu.client.ConnectToCluster.access$100(ConnectToCluster.java:47) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:323) at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:312) at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:247) at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:294) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:269) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.messageReceived(Connection.java:391) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.timeout.ReadTimeoutHandler.messageReceived(ReadTimeoutHandler.java:184) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462) at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443) at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 04:44:11.262 [ERROR - Test worker] (RetryRule.java:79) testKillLeaderBeforeCreateTable(org.apache.kudu.client.TestMasterFailover): giving up after 3 failures ]]>