java.lang.AssertionError: expected:<4> but was:<3> at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.failNotEquals(Assert.java:834) at org.junit.Assert.assertEquals(Assert.java:645) at org.junit.Assert.assertEquals(Assert.java:631) at org.apache.kudu.client.TestHybridTime.test(TestHybridTime.java:167) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) /127.29.165.254:53320] using SASL/PLAIN 09:54:25.283 [DEBUG - New I/O worker #3] (Negotiator.java:764) Authenticated connection [id: 0x836ce20b, /127.29.165.252:55143 => /127.29.165.252:36045] using SASL/PLAIN 09:54:25.283 [DEBUG - New I/O worker #2] (Negotiator.java:764) Authenticated connection [id: 0x14a76941, /127.29.165.253:42012 => /127.29.165.253:53545] using SASL/PLAIN 09:54:25.317 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1986) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 09:54:25.318 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=395), Traces: [0ms] querying master, [32ms] Sub rpc: ConnectToMaster sending RPC to server master-127.29.165.254:53320, [45ms] Sub rpc: ConnectToMaster sending RPC to server master-127.29.165.253:53545, [46ms] Sub rpc: ConnectToMaster sending RPC to server master-127.29.165.252:36045, [368ms] Sub rpc: ConnectToMaster received from server master-127.29.165.253:53545 response OK, [368ms] Sub rpc: ConnectToMaster received from server master-127.29.165.254:53320 response OK, [368ms] Sub rpc: ConnectToMaster received from server master-127.29.165.252:36045 response OK, null) after lookup 09:54:25.329 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=retry RPC@115968142 returned Deferred@1410299361(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after retry RPC@115968142), errback=(continuation of Deferred@1795643158 after retry RPC@115968142)), so the following Deferred is getting paused: Deferred@1795643158(state=PAUSED, result=Deferred@1410299361, callback=org.apache.kudu.client.AsyncKuduClient$1@173bcf90 -> org.apache.kudu.client.AsyncKuduClient$2@5d5c9801 -> wakeup thread Test worker, errback=passthrough -> passthrough -> wakeup thread Test worker) 09:54:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.329355 30460 catalog_manager.cc:1376] Servicing CreateTable request from {username='jenkins'} at 127.29.165.253:42012: 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "org.apache.kudu.client.TestHybridTime-1542822859088" 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) schema { 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) columns { 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "key" 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) type: STRING 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) is_key: true 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) is_nullable: false 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) cfile_block_size: 0 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) partition_schema { 09:54:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) range_schema { 09:54:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) columns { 09:54:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "key" 09:54:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:25.345170 30460 catalog_manager.cc:1527] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestHybridTime-1542822859088 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 09:54:25.371 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@173bcf90@389795728 returned Deferred@1926454838(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@233d189d -> (continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$1@173bcf90@389795728), errback=passthrough -> (continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$1@173bcf90@389795728)), so the following Deferred is getting paused: Deferred@1795643158(state=PAUSED, result=Deferred@1926454838, callback=org.apache.kudu.client.AsyncKuduClient$2@5d5c9801 -> wakeup thread Test worker, errback=passthrough -> wakeup thread Test worker) 09:54:25.392 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:737) Opened table f040ec008a4741718f84d3da3fc726db 09:54:25.393 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313)), so the following Deferred is getting paused: Deferred@1795643158(state=PAUSED, result=Deferred@995935064, callback=wakeup thread Test worker, errback=wakeup thread Test worker) 09:54:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.396984 30779 tablet_service.cc:772] Processing CreateTablet for tablet e4962ce16d124d89b53533a0c656e216 (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=f040ec008a4741718f84d3da3fc726db]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.398046 30779 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet e4962ce16d124d89b53533a0c656e216. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.399025 30648 tablet_service.cc:772] Processing CreateTablet for tablet e4962ce16d124d89b53533a0c656e216 (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=f040ec008a4741718f84d3da3fc726db]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.399947 30648 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet e4962ce16d124d89b53533a0c656e216. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.404536 30910 tablet_service.cc:772] Processing CreateTablet for tablet e4962ce16d124d89b53533a0c656e216 (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=f040ec008a4741718f84d3da3fc726db]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.405540 30910 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet e4962ce16d124d89b53533a0c656e216. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:25.410 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334)), so the following Deferred is getting paused: Deferred@42892618(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.417 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852)), so the following Deferred is getting paused: Deferred@1020201117(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.424827 30648 ts_tablet_manager.cc:1173] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Registered tablet (data state: TABLET_DATA_READY) 09:54:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.425846 30779 ts_tablet_manager.cc:1173] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Registered tablet (data state: TABLET_DATA_READY) 09:54:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.426129 30910 ts_tablet_manager.cc:1173] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Registered tablet (data state: TABLET_DATA_READY) 09:54:25.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.427042 31013 ts_tablet_manager.cc:1028] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Bootstrapping tablet 09:54:25.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.427851 31013 tablet_bootstrap.cc:438] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Bootstrap starting. 09:54:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.430238 31015 ts_tablet_manager.cc:1028] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Bootstrapping tablet 09:54:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.431073 31015 tablet_bootstrap.cc:438] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Bootstrap starting. 09:54:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.431134 31013 tablet_bootstrap.cc:592] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: No blocks or log segments found. Creating new log. 09:54:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.431990 31014 ts_tablet_manager.cc:1028] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Bootstrapping tablet 09:54:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.432528 31014 tablet_bootstrap.cc:438] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Bootstrap starting. 09:54:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.432991 31013 log.cc:525] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Log is configured to *not* fsync() on all Append() calls 09:54:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.434579 31015 tablet_bootstrap.cc:592] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: No blocks or log segments found. Creating new log. 09:54:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.436458 31014 tablet_bootstrap.cc:592] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: No blocks or log segments found. Creating new log. 09:54:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.437923 31014 log.cc:525] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Log is configured to *not* fsync() on all Append() calls 09:54:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.437638 31015 log.cc:525] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Log is configured to *not* fsync() on all Append() calls 09:54:25.439 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399)), so the following Deferred is getting paused: Deferred@1272510537(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.448491 31013 tablet_bootstrap.cc:438] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: No bootstrap required, opened a new log 09:54:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.448917 31013 ts_tablet_manager.cc:1045] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Time spent bootstrapping tablet: real 0.021s user 0.005s sys 0.002s 09:54:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.455255 31015 tablet_bootstrap.cc:438] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: No bootstrap required, opened a new log 09:54:25.456 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871)), so the following Deferred is getting paused: Deferred@1276113290(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.456146 31015 ts_tablet_manager.cc:1045] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Time spent bootstrapping tablet: real 0.025s user 0.005s sys 0.004s 09:54:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.456420 31014 tablet_bootstrap.cc:438] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: No bootstrap required, opened a new log 09:54:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.456852 31014 ts_tablet_manager.cc:1045] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Time spent bootstrapping tablet: real 0.025s user 0.005s sys 0.005s 09:54:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.463655 31013 raft_consensus.cc:339] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.464800 31013 raft_consensus.cc:365] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:25.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.465140 31013 raft_consensus.cc:704] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a4d5c486c43455fbd002ebf9f56802b, State: Initialized, Role: FOLLOWER 09:54:25.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.466477 31013 consensus_queue.cc:229] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.480 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194)), so the following Deferred is getting paused: Deferred@389063039(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.481878 31013 ts_tablet_manager.cc:1073] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Time spent starting tablet: real 0.033s user 0.017s sys 0.010s 09:54:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.485397 31014 raft_consensus.cc:339] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.486552 31014 raft_consensus.cc:365] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.486930 31014 raft_consensus.cc:704] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 566eac1695064dddaf4f338cb5c81f99, State: Initialized, Role: FOLLOWER 09:54:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.487939 31014 consensus_queue.cc:229] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.488467 31015 raft_consensus.cc:339] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.490046 31015 raft_consensus.cc:365] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.490504 31015 raft_consensus.cc:704] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f669cfb54c57468085d3c46c23954921, State: Initialized, Role: FOLLOWER 09:54:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.491533 31015 consensus_queue.cc:229] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.493052 31014 ts_tablet_manager.cc:1073] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Time spent starting tablet: real 0.036s user 0.022s sys 0.001s 09:54:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.500605 30974 heartbeater.cc:467] Master 127.29.165.253:53545 was elected leader, sending a full tablet report... 09:54:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:25.501152 30719 tablet.cc:1766] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.503929 31015 ts_tablet_manager.cc:1073] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Time spent starting tablet: real 0.047s user 0.019s sys 0.013s 09:54:25.516 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194) -> (continuation of Deferred@672398922 after org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194) -> (continuation of Deferred@672398922 after org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213)), so the following Deferred is getting paused: Deferred@672398922(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.556 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@3d90232@64553522 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194) -> (continuation of Deferred@672398922 after org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213) -> (continuation of Deferred@292577391 after org.apache.kudu.client.AsyncKuduClient$13@3d90232@64553522), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194) -> (continuation of Deferred@672398922 after org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213) -> (continuation of Deferred@292577391 after org.apache.kudu.client.AsyncKuduClient$13@3d90232@64553522)), so the following Deferred is getting paused: Deferred@292577391(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.571259 31020 raft_consensus.cc:471] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 09:54:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.571512 31020 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Snoozing failure detection for 1.832s (starting election) 09:54:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.571831 31020 raft_consensus.cc:493] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.573516 31020 leader_election.cc:251] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer f669cfb54c57468085d3c46c23954921 (127.29.165.195:45047) 09:54:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.574345 31020 leader_election.cc:251] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 2a4d5c486c43455fbd002ebf9f56802b (127.29.165.193:56146) 09:54:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:25.524770 30850 tablet.cc:1766] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.600411 30930 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "e4962ce16d124d89b53533a0c656e216" candidate_uuid: "566eac1695064dddaf4f338cb5c81f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f669cfb54c57468085d3c46c23954921" is_pre_election: true 09:54:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.600976 30930 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Snoozing failure detection for 1.828s (vote granted) 09:54:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.601217 30930 raft_consensus.cc:2352] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 566eac1695064dddaf4f338cb5c81f99 in term 0. 09:54:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.602387 30731 leader_election.cc:387] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 pre-election: Vote granted by peer f669cfb54c57468085d3c46c23954921 (127.29.165.195:45047) 09:54:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.602622 30731 leader_election.cc:278] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 09:54:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.603332 31020 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Snoozing failure detection for 1.924s (election complete) 09:54:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.603596 31020 raft_consensus.cc:2646] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Leader pre-election won for term 1 09:54:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.603828 31020 raft_consensus.cc:471] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 09:54:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.604017 31020 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Snoozing failure detection for 2.095s (starting election) 09:54:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.603961 30668 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "e4962ce16d124d89b53533a0c656e216" candidate_uuid: "566eac1695064dddaf4f338cb5c81f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a4d5c486c43455fbd002ebf9f56802b" is_pre_election: true 09:54:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.604216 31020 raft_consensus.cc:2892] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 0 FOLLOWER]: Advancing to term 1 09:54:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.604521 30668 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Snoozing failure detection for 1.823s (vote granted) 09:54:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.604815 30668 raft_consensus.cc:2352] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 566eac1695064dddaf4f338cb5c81f99 in term 0. 09:54:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.606230 30729 leader_election.cc:387] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 pre-election: Vote granted by peer 2a4d5c486c43455fbd002ebf9f56802b (127.29.165.193:56146) 09:54:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.607718 31020 raft_consensus.cc:493] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.608479 31020 leader_election.cc:251] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 election: Requesting vote from peer f669cfb54c57468085d3c46c23954921 (127.29.165.195:45047) 09:54:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.608892 31020 leader_election.cc:251] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 election: Requesting vote from peer 2a4d5c486c43455fbd002ebf9f56802b (127.29.165.193:56146) 09:54:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.610111 30930 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "e4962ce16d124d89b53533a0c656e216" candidate_uuid: "566eac1695064dddaf4f338cb5c81f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f669cfb54c57468085d3c46c23954921" 09:54:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.610442 30930 raft_consensus.cc:2892] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 0 FOLLOWER]: Advancing to term 1 09:54:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.610745 30930 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Snoozing failure detection for 1.672s (vote granted) 09:54:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.611068 30668 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "e4962ce16d124d89b53533a0c656e216" candidate_uuid: "566eac1695064dddaf4f338cb5c81f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a4d5c486c43455fbd002ebf9f56802b" 09:54:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.611474 30668 raft_consensus.cc:2892] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 0 FOLLOWER]: Advancing to term 1 09:54:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.611773 30668 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b: Snoozing failure detection for 1.800s (vote granted) 09:54:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.614125 30930 raft_consensus.cc:2352] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 566eac1695064dddaf4f338cb5c81f99 in term 1. 09:54:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.615094 30731 leader_election.cc:387] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 election: Vote granted by peer f669cfb54c57468085d3c46c23954921 (127.29.165.195:45047) 09:54:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.615337 30731 leader_election.cc:278] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 09:54:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.615988 31020 raft_consensus.cc:2840] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99: Snoozing failure detection for 1.986s (election complete) 09:54:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.616230 31020 raft_consensus.cc:2646] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 1 FOLLOWER]: Leader election won for term 1 09:54:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.616341 30668 raft_consensus.cc:2352] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 566eac1695064dddaf4f338cb5c81f99 in term 1. 09:54:25.617 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@142ece33@338611763 returned Deferred@995935064(state=PENDING, result=null, callback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194) -> (continuation of Deferred@672398922 after org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213) -> (continuation of Deferred@292577391 after org.apache.kudu.client.AsyncKuduClient$13@3d90232@64553522) -> (continuation of Deferred@603489106 after org.apache.kudu.client.AsyncKuduClient$13@142ece33@338611763), errback=(continuation of Deferred@1795643158 after org.apache.kudu.client.AsyncKuduClient$2@5d5c9801@1566349313) -> (continuation of Deferred@42892618 after org.apache.kudu.client.AsyncKuduClient$13@70eecd5e@1894698334) -> (continuation of Deferred@1020201117 after org.apache.kudu.client.AsyncKuduClient$13@615441ec@1632911852) -> (continuation of Deferred@1272510537 after org.apache.kudu.client.AsyncKuduClient$13@3a4da667@978167399) -> (continuation of Deferred@1276113290 after org.apache.kudu.client.AsyncKuduClient$13@8408bff@138447871) -> (continuation of Deferred@389063039 after org.apache.kudu.client.AsyncKuduClient$13@1ba618f2@463870194) -> (continuation of Deferred@672398922 after org.apache.kudu.client.AsyncKuduClient$13@1ba3cf15@463720213) -> (continuation of Deferred@292577391 after org.apache.kudu.client.AsyncKuduClient$13@3d90232@64553522) -> (continuation of Deferred@603489106 after org.apache.kudu.client.AsyncKuduClient$13@142ece33@338611763)), so the following Deferred is getting paused: Deferred@603489106(state=PAUSED, result=Deferred@995935064, callback=, errback=) 09:54:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.617895 30729 leader_election.cc:387] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [CANDIDATE]: Term 1 election: Vote granted by peer 2a4d5c486c43455fbd002ebf9f56802b (127.29.165.193:56146) 09:54:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.618487 31020 raft_consensus.cc:666] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [term 1 LEADER]: Becoming Leader. State: Replica: 566eac1695064dddaf4f338cb5c81f99, State: Running, Role: LEADER 09:54:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.619357 31020 consensus_queue.cc:206] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } } 09:54:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.628620 30459 catalog_manager.cc:3781] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 reported cstate change: term changed from 0 to 1, leader changed from to 566eac1695064dddaf4f338cb5c81f99 (127.29.165.194). New cstate: current_term: 1 leader_uuid: "566eac1695064dddaf4f338cb5c81f99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "566eac1695064dddaf4f338cb5c81f99" member_type: VOTER last_known_addr { host: "127.29.165.194" port: 38085 } health_report { overall_health: HEALTHY } } } 09:54:25.694 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1986) Learned about tablet e4962ce16d124d89b53533a0c656e216 for table 'org.apache.kudu.client.TestHybridTime-1542822859088' with partition [, ) 09:54:25.702 [DEBUG - New I/O worker #2] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=e4962ce16d124d89b53533a0c656e216@[2a4d5c486c43455fbd002ebf9f56802b(127.29.165.193:56146),566eac1695064dddaf4f338cb5c81f99(127.29.165.194:38085)[L],f669cfb54c57468085d3c46c23954921(127.29.165.195:45047)]}] 09:54:25.703 [DEBUG - New I/O worker #2] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=18), Traces: [0ms] querying master, [0ms] Sub rpc: GetTableLocations sending RPC to server master-127.29.165.253:53545, [5ms] Sub rpc: GetTableLocations received from server master-127.29.165.253:53545 response OK, null) after lookup 09:54:25.710 [DEBUG - New I/O worker #2] (Deferred.java:1330) callback=retry RPC@1530775730 returned Deferred@937462326(state=PENDING, result=null, callback=(continuation of Deferred@1966998720 after retry RPC@1530775730), errback=(continuation of Deferred@1966998720 after retry RPC@1530775730)), so the following Deferred is getting paused: Deferred@1966998720(state=PAUSED, result=Deferred@937462326, callback=org.apache.kudu.client.AsyncKuduSession$2@42ff7d9b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7732552e -> wakeup thread Time-limited test) 09:54:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:25.648483 30982 tablet.cc:1766] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:25.773 [DEBUG - New I/O worker #4] (Negotiator.java:764) Authenticated connection [id: 0x3077bbec, /127.29.165.194:45376 => /127.29.165.194:38085] using TOKEN/null 09:54:25.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:25.781131 31020 consensus_queue.cc:745] Safe time advancement without writes is disabled. Snapshot reads on non-leader replicas may stall if there are no writes in progress. 09:54:25.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.783471 30930 raft_consensus.cc:1180] T e4962ce16d124d89b53533a0c656e216 P f669cfb54c57468085d3c46c23954921 [term 1 FOLLOWER]: Refusing update from remote peer 566eac1695064dddaf4f338cb5c81f99: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.783850 30668 raft_consensus.cc:1180] T e4962ce16d124d89b53533a0c656e216 P 2a4d5c486c43455fbd002ebf9f56802b [term 1 FOLLOWER]: Refusing update from remote peer 566eac1695064dddaf4f338cb5c81f99: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:25.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.785084 31020 consensus_queue.cc:976] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f669cfb54c57468085d3c46c23954921" member_type: VOTER last_known_addr { host: "127.29.165.195" port: 45047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:25.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.785897 31024 consensus_queue.cc:976] T e4962ce16d124d89b53533a0c656e216 P 566eac1695064dddaf4f338cb5c81f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a4d5c486c43455fbd002ebf9f56802b" member_type: VOTER last_known_addr { host: "127.29.165.193" port: 56146 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.803272 31027 mvcc.cc:203] Tried to move safe_time back from 6319402458234720256 to 6319402457583124480. Current Snapshot: MvccSnapshot[committed={T|T < 6319402458234720256}] 09:54:25.831 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[0]: Wed Nov 21 09:54:25 PST 2018 Logical value: 0 09:54:25.841 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[1]: Wed Nov 21 09:54:30 PST 2018 Logical value: 1 09:54:25.853 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[2]: Wed Nov 21 09:54:30 PST 2018 Logical value: 4 09:54:25.854 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@1551450379(state=PENDING, result=null, callback=, errback=)} 09:54:25.858 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@cf37e63@217284195 returned Deferred@926328933(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@306284312 after org.apache.kudu.client.AsyncKuduSession$1@cf37e63@217284195), errback=passthrough -> (continuation of Deferred@306284312 after org.apache.kudu.client.AsyncKuduSession$1@cf37e63@217284195)), so the following Deferred is getting paused: Deferred@306284312(state=PAUSED, result=Deferred@926328933, callback=, errback=) 09:54:25.867 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[0]: Wed Nov 21 09:54:30 PST 2018 Logical value: 5 09:54:25.867 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@443988065(state=PENDING, result=null, callback=, errback=)} 09:54:25.868 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@de1dd2a@232906026 returned Deferred@261503347(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@1551450379 after org.apache.kudu.client.AsyncKuduSession$1@de1dd2a@232906026), errback=passthrough -> (continuation of Deferred@1551450379 after org.apache.kudu.client.AsyncKuduSession$1@de1dd2a@232906026)), so the following Deferred is getting paused: Deferred@1551450379(state=PAUSED, result=Deferred@261503347, callback=, errback=) 09:54:25.878 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[1]: Wed Nov 21 09:54:30 PST 2018 Logical value: 6 09:54:25.879 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@292624094(state=PENDING, result=null, callback=, errback=)} 09:54:25.879 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@3e6b1121@1047204129 returned Deferred@1038123851(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@443988065 after org.apache.kudu.client.AsyncKuduSession$1@3e6b1121@1047204129), errback=passthrough -> (continuation of Deferred@443988065 after org.apache.kudu.client.AsyncKuduSession$1@3e6b1121@1047204129)), so the following Deferred is getting paused: Deferred@443988065(state=PAUSED, result=Deferred@1038123851, callback=, errback=) 09:54:25.893 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[2]: Wed Nov 21 09:54:30 PST 2018 Logical value: 7 09:54:25.935 [DEBUG - New I/O worker #4] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=e4962ce16d124d89b53533a0c656e216, scannerId=null, scanRequestTimeout=30000) 09:54:25.943 [DEBUG - New I/O worker #4] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 1 rows, more = false, responseScanTimestamp = 6319402478714720256) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=e4962ce16d124d89b53533a0c656e216, scannerId=null, scanRequestTimeout=30000) 09:54:25.948 [DEBUG - New I/O worker #4] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 3 rows, more = false, responseScanTimestamp = 6319402478714720261) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=e4962ce16d124d89b53533a0c656e216, scannerId=null, scanRequestTimeout=30000) 09:54:25.956 [DEBUG - New I/O worker #1] (Connection.java:688) [peer master-127.29.165.254:53320(127.29.165.254:53320)] cleaning up while in state READY due to: connection disconnected 09:54:25.958 [DEBUG - New I/O worker #2] (Connection.java:688) [peer master-127.29.165.253:53545(127.29.165.253:53545)] cleaning up while in state READY due to: connection disconnected 09:54:25.962 [DEBUG - New I/O worker #3] (Connection.java:688) [peer master-127.29.165.252:36045(127.29.165.252:36045)] cleaning up while in state READY due to: connection disconnected 09:54:25.963 [DEBUG - New I/O worker #4] (Connection.java:688) [peer 566eac1695064dddaf4f338cb5c81f99(127.29.165.194:38085)] cleaning up while in state READY due to: connection disconnected 09:54:25.964 [DEBUG - Test worker] (AsyncKuduClient.java:2132) Releasing all remaining resources 09:54:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:25.965945 30359 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 30559 09:54:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.003358 30376 connection.cc:512] server connection from 127.29.165.193:33046 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.193:33046): Connection reset by peer (error 104) 09:54:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.003346 30444 connection.cc:512] server connection from 127.29.165.193:59832 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.193:59832): Connection reset by peer (error 104) 09:54:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:26.004331 30359 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 30712 09:54:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.004284 30508 connection.cc:512] server connection from 127.29.165.193:41203 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.193:41203): Connection reset by peer (error 104) 09:54:26.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.024953 30506 connection.cc:512] server connection from 127.29.165.194:36418 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.194:36418): Connection reset by peer (error 104) 09:54:26.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.025144 30378 connection.cc:512] server connection from 127.29.165.194:58634 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.194:58634): Connection reset by peer (error 104) 09:54:26.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.025916 30440 connection.cc:512] server connection from 127.29.165.194:52167 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.194:52167): Connection reset by peer (error 104) 09:54:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:26.026235 30359 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 30846 09:54:26.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.045907 30376 connection.cc:512] server connection from 127.29.165.195:51536 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.195:51536): Connection reset by peer (error 104) 09:54:26.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.045944 30441 connection.cc:512] server connection from 127.29.165.195:42046 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.195:42046): Connection reset by peer (error 104) 09:54:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:26.046707 30506 connection.cc:512] server connection from 127.29.165.195:39309 recv error: Network error: failed to read from TLS socket (remote: 127.29.165.195:39309): Connection reset by peer (error 104) 09:54:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:26.046838 30359 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 30368 09:54:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:26.066781 30359 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 30430 09:54:26.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:26.087229 30359 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 30494 09:54:27.127 [ERROR - Test worker] (RetryRule.java:76) test(org.apache.kudu.client.TestHybridTime): failed run 1 java.lang.AssertionError: expected:<4> but was:<3> at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.failNotEquals(Assert.java:834) at org.junit.Assert.assertEquals(Assert.java:645) at org.junit.Assert.assertEquals(Assert.java:631) at org.apache.kudu.client.TestHybridTime.test(TestHybridTime.java:167) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 09:54:27.129 [INFO - Test worker] (KuduTestHarness.java:137) Creating a new MiniKuduCluster... 09:54:27.129 [INFO - Test worker] (MiniKuduCluster.java:146) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 09:54:27.130 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 09:54:27.130 [INFO - Test worker] (MiniKuduCluster.java:197) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 09:54:27.131 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409" extra_tserver_flags: "--safe_time_advancement_without_writes=false" hms_mode: NONE mini_kdc_options { } } 09:54:27.327 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: 09:54:27.327 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: start_cluster { } 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.351788 31042 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/wal 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=localhost 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.190:60740 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.190 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/logs 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data/info.pb 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 with env {} 09:54:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.521045 31050 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.521502 31050 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.532023 31050 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 09:54:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.532181 31050 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.532513 31050 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 09:54:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.532690 31050 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 09:54:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.553298 31050 master_main.cc:73] Master server non-default flags: 09:54:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data 09:54:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/wal 09:54:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.190:60740 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data/info.pb 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.190 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/logs 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Master server version: 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.567929 31050 master_main.cc:80] Initializing master server... 09:54:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.570891 31050 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 441527us 09:54:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.571730 31050 fs_manager.cc:263] Metadata directory not provided 09:54:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.571914 31050 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.572273 31050 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.572403 31050 server_base.cc:433] Attempting to create new FS layout instead 09:54:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.576539 31050 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data/instance: 09:54:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "84cc603334ae465493a2e46b51d40732" 09:54:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:27 on quasar-itaafk-1.vpc.cloudera.com" 09:54:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.577431 31050 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/wal/instance: 09:54:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "84cc603334ae465493a2e46b51d40732" 09:54:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:27 on quasar-itaafk-1.vpc.cloudera.com" 09:54:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.592192 31050 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.007s sys 0.005s 09:54:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.593559 31050 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.594866 31050 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.603144 31050 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s 09:54:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.603286 31050 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/wal 09:54:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "84cc603334ae465493a2e46b51d40732" 09:54:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:27 on quasar-itaafk-1.vpc.cloudera.com" 09:54:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.603603 31050 fs_report.cc:352] FS layout report 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/wal 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/wal 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data/data 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.772609 31050 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.775281 31050 master_main.cc:83] Starting Master server... 09:54:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.867122 31050 rpc_server.cc:205] RPC server started. Bound to: 127.30.80.190:60740 09:54:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.868250 31050 webserver.cc:175] Starting webserver on 127.30.80.190:0 09:54:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.868367 31050 webserver.cc:186] Document root disabled 09:54:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.870504 31050 webserver.cc:313] Webserver started. Bound to: http://127.30.80.190:34995/ 09:54:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.871459 31050 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-0/data/info.pb 09:54:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.872473 31042 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 31050 09:54:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.873723 31042 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/wal 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=localhost 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.189:34393 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.189 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/logs 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data/info.pb 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:27.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:27.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 with env {} 09:54:27.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.876060 31111 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.890851 31111 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } has no permanent_uuid. Determining permanent_uuid... 09:54:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.912678 31111 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } has no permanent_uuid. Determining permanent_uuid... 09:54:27.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.914649 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.189:34393: Network error: Client connection negotiation failed: client connection to 127.30.80.189:34393: connect: Connection refused (error 111) 09:54:27.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.934146 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } attempt: 1 09:54:27.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.935575 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.189:34393: Network error: Client connection negotiation failed: client connection to 127.30.80.189:34393: connect: Connection refused (error 111) 09:54:27.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:27.983953 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } attempt: 2 09:54:27.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:27.985283 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.189:34393: Network error: Client connection negotiation failed: client connection to 127.30.80.189:34393: connect: Connection refused (error 111) 09:54:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.046620 31112 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.047045 31112 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.057121 31112 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 09:54:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.057250 31112 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.057374 31112 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 09:54:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.057504 31112 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.077875 31112 master_main.cc:73] Master server non-default flags: 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/wal 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.189:34393 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data/info.pb 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.189 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/logs 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Master server version: 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.092203 31112 master_main.cc:80] Initializing master server... 09:54:28.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.094648 31112 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 442027us 09:54:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.095468 31112 fs_manager.cc:263] Metadata directory not provided 09:54:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.095600 31112 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.095937 31112 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.096055 31112 server_base.cc:433] Attempting to create new FS layout instead 09:54:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.096663 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } attempt: 3 09:54:28.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.098026 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.189:34393: Network error: Client connection negotiation failed: client connection to 127.30.80.189:34393: connect: Connection refused (error 111) 09:54:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.100026 31112 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data/instance: 09:54:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "890d26b4bbaf44d39d7e7107628b0b14" 09:54:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:28 on quasar-itaafk-1.vpc.cloudera.com" 09:54:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.100839 31112 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/wal/instance: 09:54:28.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "890d26b4bbaf44d39d7e7107628b0b14" 09:54:28.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:28 on quasar-itaafk-1.vpc.cloudera.com" 09:54:28.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.115613 31112 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.008s sys 0.005s 09:54:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.117027 31112 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:28.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.118341 31112 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:28.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.126590 31112 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.126721 31112 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/wal 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "890d26b4bbaf44d39d7e7107628b0b14" 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:28 on quasar-itaafk-1.vpc.cloudera.com" 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.127085 31112 fs_report.cc:352] FS layout report 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/wal 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/wal 09:54:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data/data 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:28.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:28.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.274369 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } attempt: 4 09:54:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.275749 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.189:34393: Network error: Client connection negotiation failed: client connection to 127.30.80.189:34393: connect: Connection refused (error 111) 09:54:28.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.295456 31112 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.298169 31112 master_main.cc:83] Starting Master server... 09:54:28.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.393244 31112 rpc_server.cc:205] RPC server started. Bound to: 127.30.80.189:34393 09:54:28.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.394381 31112 webserver.cc:175] Starting webserver on 127.30.80.189:0 09:54:28.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.394498 31112 webserver.cc:186] Document root disabled 09:54:28.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.396984 31112 webserver.cc:313] Webserver started. Bound to: http://127.30.80.189:43500/ 09:54:28.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.397948 31112 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-1/data/info.pb 09:54:28.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.402143 31175 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.403674 31042 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 31112 09:54:28.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.404939 31042 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/wal 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=localhost 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.188:49312 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.188 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/logs 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data/info.pb 09:54:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 with env {} 09:54:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.418597 31175 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } has no permanent_uuid. Determining permanent_uuid... 09:54:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.435621 31175 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } has no permanent_uuid. Determining permanent_uuid... 09:54:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.447815 31175 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } has no permanent_uuid. Determining permanent_uuid... 09:54:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.449177 31175 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.468586 31175 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 1 09:54:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.470098 31175 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.518447 31175 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 2 09:54:28.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.519969 31175 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.576047 31176 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.576432 31176 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.579090 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } attempt: 5 09:54:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.586308 31111 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } has no permanent_uuid. Determining permanent_uuid... 09:54:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.588008 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.589452 31176 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 09:54:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.589587 31176 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.589690 31176 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 09:54:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.589833 31176 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.609954 31176 master_main.cc:73] Master server non-default flags: 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/wal 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.188:49312 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data/info.pb 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.188 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/logs 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Master server version: 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.623482 31176 master_main.cc:80] Initializing master server... 09:54:28.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.625877 31176 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 442027us 09:54:28.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.626324 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 1 09:54:28.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.626626 31176 fs_manager.cc:263] Metadata directory not provided 09:54:28.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.626775 31176 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.627146 31176 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.627319 31176 server_base.cc:433] Attempting to create new FS layout instead 09:54:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.628101 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.631299 31175 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 3 09:54:28.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.631433 31176 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data/instance: 09:54:28.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "c718bc7374da41ceb8a28d61af20bfba" 09:54:28.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:28 on quasar-itaafk-1.vpc.cloudera.com" 09:54:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.632331 31176 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/wal/instance: 09:54:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "c718bc7374da41ceb8a28d61af20bfba" 09:54:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:28 on quasar-itaafk-1.vpc.cloudera.com" 09:54:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.632709 31175 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.646477 31176 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.008s sys 0.004s 09:54:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.647814 31176 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:28.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.649086 31176 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:28.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.657241 31176 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s 09:54:28.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.657378 31176 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/wal 09:54:28.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "c718bc7374da41ceb8a28d61af20bfba" 09:54:28.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:28 on quasar-itaafk-1.vpc.cloudera.com" 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.657680 31176 fs_report.cc:352] FS layout report 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/wal 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/wal 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data/data 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.676426 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 2 09:54:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.678268 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.779001 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 3 09:54:28.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.780594 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.809073 31175 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 4 09:54:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.810542 31175 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.832072 31176 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.834503 31176 master_main.cc:83] Starting Master server... 09:54:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.920969 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 4 09:54:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:28.922394 31111 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.80.188:49312: Network error: Client connection negotiation failed: client connection to 127.30.80.188:49312: connect: Connection refused (error 111) 09:54:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.925985 31176 rpc_server.cc:205] RPC server started. Bound to: 127.30.80.188:49312 09:54:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.926863 31176 webserver.cc:175] Starting webserver on 127.30.80.188:0 09:54:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.926965 31176 webserver.cc:186] Document root disabled 09:54:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.928859 31176 webserver.cc:313] Webserver started. Bound to: http://127.30.80.188:47148/ 09:54:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.929651 31176 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/master-2/data/info.pb 09:54:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.931422 31042 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 31176 09:54:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.932526 31042 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/wal 09:54:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data 09:54:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.129:0 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.80.129 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.129 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/logs 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data/info.pb 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false with env {} 09:54:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.934257 31240 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.953667 31240 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } has no permanent_uuid. Determining permanent_uuid... 09:54:28.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.975513 31240 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } has no permanent_uuid. Determining permanent_uuid... 09:54:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:28.984843 31240 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } has no permanent_uuid. Determining permanent_uuid... 09:54:29.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.005007 31240 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Bootstrap starting. 09:54:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.009471 31240 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: No blocks or log segments found. Creating new log. 09:54:29.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.011831 31240 log.cc:525] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Log is configured to *not* fsync() on all Append() calls 09:54:29.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.017383 31240 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: No bootstrap required, opened a new log 09:54:29.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.037041 31240 raft_consensus.cc:339] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.038223 31240 raft_consensus.cc:365] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.038640 31240 raft_consensus.cc:704] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c718bc7374da41ceb8a28d61af20bfba, State: Initialized, Role: FOLLOWER 09:54:29.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.039595 31240 consensus_queue.cc:229] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.043109 31240 sys_catalog.cc:337] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.044157 31240 sys_catalog.cc:340] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.045928 31240 sys_catalog.cc:424] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: configured and running, proceeding with master startup. 09:54:29.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.113842 31175 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 5 09:54:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.120811 31241 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.121770 31241 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.129139 31175 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: Bootstrap starting. 09:54:29.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.131062 31241 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:29.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.131491 31241 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.80.129 09:54:29.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.140861 31175 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: No blocks or log segments found. Creating new log. 09:54:29.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.142837 31175 log.cc:525] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: Log is configured to *not* fsync() on all Append() calls 09:54:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.151921 31175 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: No bootstrap required, opened a new log 09:54:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.153468 31241 tablet_server_main.cc:78] Tablet server non-default flags: 09:54:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false 09:54:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data 09:54:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/wal 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.129:0 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data/info.pb 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.129 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.80.129 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/logs 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Tablet server version: 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.168891 31241 tablet_server_main.cc:85] Initializing tablet server... 09:54:29.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.044062 31246 sys_catalog.cc:337] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.171371 31246 sys_catalog.cc:340] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.172163 31241 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 442527us 09:54:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.172804 31241 fs_manager.cc:263] Metadata directory not provided 09:54:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.172976 31241 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.173455 31241 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.173727 31241 server_base.cc:433] Attempting to create new FS layout instead 09:54:29.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.175956 31175 raft_consensus.cc:339] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.176429 31175 raft_consensus.cc:365] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:29.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.176650 31175 raft_consensus.cc:704] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 890d26b4bbaf44d39d7e7107628b0b14, State: Initialized, Role: FOLLOWER 09:54:29.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.177330 31175 consensus_queue.cc:229] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.179953 31175 sys_catalog.cc:337] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.180958 31175 sys_catalog.cc:340] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.181346 31241 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data/instance: 09:54:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" 09:54:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:29 on quasar-itaafk-1.vpc.cloudera.com" 09:54:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.182462 31241 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/wal/instance: 09:54:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" 09:54:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:29 on quasar-itaafk-1.vpc.cloudera.com" 09:54:29.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.183089 31175 sys_catalog.cc:424] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: configured and running, proceeding with master startup. 09:54:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.187712 31111 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } attempt: 5 09:54:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.188330 31246 raft_consensus.cc:471] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 09:54:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.188614 31246 raft_consensus.cc:2840] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Snoozing failure detection for 1.983s (starting election) 09:54:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.188947 31246 raft_consensus.cc:493] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.190263 31246 leader_election.cc:251] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 84cc603334ae465493a2e46b51d40732 (127.30.80.190:60740) 09:54:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.190912 31246 leader_election.cc:251] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 890d26b4bbaf44d39d7e7107628b0b14 (127.30.80.189:34393) 09:54:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.206651 31111 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: Bootstrap starting. 09:54:29.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.210502 31111 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: No blocks or log segments found. Creating new log. 09:54:29.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.212399 31111 log.cc:525] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: Log is configured to *not* fsync() on all Append() calls 09:54:29.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.087790 31196 tablet.cc:1766] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:29.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.222234 31111 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: No bootstrap required, opened a new log 09:54:29.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.229800 31176 master_main.cc:86] Master server successfully started. 09:54:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.236160 31247 catalog_manager.cc:1094] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: acquiring CA information for follower catalog manager: Not found: root CA entry not found 09:54:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.236393 31247 catalog_manager.cc:584] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 09:54:29.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.239662 31241 fs_manager.cc:503] Time spent creating directory manager: real 0.057s user 0.015s sys 0.004s 09:54:29.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.241430 31241 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.242843 31241 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.253511 31111 raft_consensus.cc:339] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.254914 31111 raft_consensus.cc:365] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:29.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.255463 31111 raft_consensus.cc:704] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84cc603334ae465493a2e46b51d40732, State: Initialized, Role: FOLLOWER 09:54:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.256808 31111 consensus_queue.cc:229] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.267560 31111 sys_catalog.cc:337] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.269038 31111 sys_catalog.cc:340] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.271924 31241 fs_manager.cc:419] Time spent opening block manager: real 0.011s user 0.001s sys 0.001s 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.271615 31088 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c718bc7374da41ceb8a28d61af20bfba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84cc603334ae465493a2e46b51d40732" is_pre_election: true 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.272104 31241 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/wal 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:29 on quasar-itaafk-1.vpc.cloudera.com" 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.272305 31088 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: Snoozing failure detection for 1.519s (vote granted) 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.272418 31241 fs_report.cc:352] FS layout report 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/wal 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/wal 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data/data 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.272593 31088 raft_consensus.cc:2352] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c718bc7374da41ceb8a28d61af20bfba in term 0. 09:54:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.273658 31188 leader_election.cc:387] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 pre-election: Vote granted by peer 84cc603334ae465493a2e46b51d40732 (127.30.80.190:60740) 09:54:29.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.273991 31188 leader_election.cc:278] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 09:54:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.274688 31246 raft_consensus.cc:2840] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Snoozing failure detection for 1.681s (election complete) 09:54:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.274962 31246 raft_consensus.cc:2646] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Leader pre-election won for term 1 09:54:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.275177 31246 raft_consensus.cc:471] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 09:54:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.275377 31246 raft_consensus.cc:2840] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Snoozing failure detection for 2.089s (starting election) 09:54:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.275583 31246 raft_consensus.cc:2892] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 0 FOLLOWER]: Advancing to term 1 09:54:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.180845 31253 sys_catalog.cc:337] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.278157 31253 sys_catalog.cc:340] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.281400 31246 raft_consensus.cc:493] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.282409 31246 leader_election.cc:251] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 election: Requesting vote from peer 84cc603334ae465493a2e46b51d40732 (127.30.80.190:60740) 09:54:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.273345 31111 sys_catalog.cc:424] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: configured and running, proceeding with master startup. 09:54:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.284938 31246 leader_election.cc:251] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 election: Requesting vote from peer 890d26b4bbaf44d39d7e7107628b0b14 (127.30.80.189:34393) 09:54:29.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.291023 31151 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c718bc7374da41ceb8a28d61af20bfba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "890d26b4bbaf44d39d7e7107628b0b14" 09:54:29.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.291757 31151 raft_consensus.cc:2892] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 0 FOLLOWER]: Advancing to term 1 09:54:29.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.292136 31151 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: Snoozing failure detection for 1.984s (vote granted) 09:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.304975 31131 tablet.cc:1766] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.307273 31151 raft_consensus.cc:2352] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c718bc7374da41ceb8a28d61af20bfba in term 1. 09:54:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.313149 31191 leader_election.cc:387] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 election: Vote granted by peer 890d26b4bbaf44d39d7e7107628b0b14 (127.30.80.189:34393) 09:54:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.313396 31191 leader_election.cc:278] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 09:54:29.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.314128 31246 raft_consensus.cc:2840] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Snoozing failure detection for 2.186s (election complete) 09:54:29.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.314369 31246 raft_consensus.cc:2646] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 1 FOLLOWER]: Leader election won for term 1 09:54:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.320257 31246 raft_consensus.cc:666] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [term 1 LEADER]: Becoming Leader. State: Replica: c718bc7374da41ceb8a28d61af20bfba, State: Running, Role: LEADER 09:54:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.329802 31246 consensus_queue.cc:206] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } 09:54:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.192441 31152 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c718bc7374da41ceb8a28d61af20bfba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "890d26b4bbaf44d39d7e7107628b0b14" is_pre_election: true 09:54:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.349200 31254 catalog_manager.cc:526] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting 09:54:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.349582 31152 raft_consensus.cc:2265] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate c718bc7374da41ceb8a28d61af20bfba in term 1. Re-sending same reply. 09:54:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.350621 31191 leader_election.cc:387] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 pre-election: Vote granted by peer 890d26b4bbaf44d39d7e7107628b0b14 (127.30.80.189:34393) 09:54:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.357010 31112 master_main.cc:86] Master server successfully started. 09:54:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.357197 31262 sys_catalog.cc:337] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: SysCatalogTable state changed. Reason: New leader c718bc7374da41ceb8a28d61af20bfba. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.358749 31262 sys_catalog.cc:340] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: This master's current role is: LEADER 09:54:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.373447 31267 catalog_manager.cc:1036] Loading table and tablet metadata into memory... 09:54:29.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.377380 31267 catalog_manager.cc:1047] Initializing Kudu internal certificate authority... 09:54:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:29.279531 31067 tablet.cc:1766] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:29.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.283659 31088 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c718bc7374da41ceb8a28d61af20bfba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84cc603334ae465493a2e46b51d40732" 09:54:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.447715 31088 raft_consensus.cc:2892] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 0 FOLLOWER]: Advancing to term 1 09:54:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.448065 31088 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: Snoozing failure detection for 1.929s (vote granted) 09:54:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.447779 31259 sys_catalog.cc:337] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.449203 31259 sys_catalog.cc:340] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.454825 31050 master_main.cc:86] Master server successfully started. 09:54:29.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.449362 31261 catalog_manager.cc:526] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting 09:54:29.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.454100 31088 raft_consensus.cc:2352] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c718bc7374da41ceb8a28d61af20bfba in term 1. 09:54:29.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.456465 31188 leader_election.cc:387] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [CANDIDATE]: Term 1 election: Vote granted by peer 84cc603334ae465493a2e46b51d40732 (127.30.80.190:60740) 09:54:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.465910 31088 raft_consensus.cc:1180] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [term 1 FOLLOWER]: Refusing update from remote peer c718bc7374da41ceb8a28d61af20bfba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.466135 31152 raft_consensus.cc:1180] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [term 1 FOLLOWER]: Refusing update from remote peer c718bc7374da41ceb8a28d61af20bfba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.467219 31246 consensus_queue.cc:976] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [LEADER]: Connected to new peer: Peer: permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:29.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.467847 31262 consensus_queue.cc:976] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [LEADER]: Connected to new peer: Peer: permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.494992 31259 sys_catalog.cc:337] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c718bc7374da41ceb8a28d61af20bfba. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.495649 31259 sys_catalog.cc:340] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.499975 31269 mvcc.cc:203] Tried to move safe_time back from 6319402473318887424 to 6319402472784539648. Current Snapshot: MvccSnapshot[committed={T|T < 6319402473318887424}] 09:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.501163 31246 sys_catalog.cc:337] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.501878 31246 sys_catalog.cc:340] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: This master's current role is: LEADER 09:54:29.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.504824 31267 catalog_manager.cc:931] Generated new certificate authority record 09:54:29.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.505645 31259 sys_catalog.cc:337] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.506317 31259 sys_catalog.cc:340] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.507616 31267 catalog_manager.cc:1056] Loading token signing keys... 09:54:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.516924 31253 sys_catalog.cc:337] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c718bc7374da41ceb8a28d61af20bfba. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.517626 31253 sys_catalog.cc:340] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.519538 31246 sys_catalog.cc:337] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.520189 31246 sys_catalog.cc:340] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba [sys.catalog]: This master's current role is: LEADER 09:54:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.524762 31253 sys_catalog.cc:337] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c718bc7374da41ceb8a28d61af20bfba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "84cc603334ae465493a2e46b51d40732" member_type: VOTER last_known_addr { host: "127.30.80.190" port: 60740 } } peers { permanent_uuid: "890d26b4bbaf44d39d7e7107628b0b14" member_type: VOTER last_known_addr { host: "127.30.80.189" port: 34393 } } peers { permanent_uuid: "c718bc7374da41ceb8a28d61af20bfba" member_type: VOTER last_known_addr { host: "127.30.80.188" port: 49312 } } } 09:54:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.525465 31253 sys_catalog.cc:340] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14 [sys.catalog]: This master's current role is: FOLLOWER 09:54:29.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.549821 31267 catalog_manager.cc:4085] T 00000000000000000000000000000000 P c718bc7374da41ceb8a28d61af20bfba: Generated new TSK 0 09:54:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.617452 31241 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.621762 31241 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 09:54:29.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.624255 31241 tablet_server_main.cc:90] Starting tablet server... 09:54:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.894790 31241 rpc_server.cc:205] RPC server started. Bound to: 127.30.80.129:40692 09:54:29.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.896018 31241 webserver.cc:175] Starting webserver on 127.30.80.129:0 09:54:29.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.896133 31241 webserver.cc:186] Document root disabled 09:54:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.897889 31241 webserver.cc:313] Webserver started. Bound to: http://127.30.80.129:40574/ 09:54:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.898687 31241 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-0/data/info.pb 09:54:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.907503 31042 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 31241 09:54:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.908756 31042 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/wal 09:54:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.130:0 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.80.130 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.130 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/logs 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data/info.pb 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false with env {} 09:54:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.933404 31241 tablet_server_main.cc:93] Tablet server successfully started. 09:54:29.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.952839 31387 heartbeater.cc:346] Connected to a master server at 127.30.80.189:34393 09:54:29.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.953181 31387 heartbeater.cc:426] Registering TS with master... 09:54:29.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.953886 31387 heartbeater.cc:475] Master 127.30.80.189:34393 requested a full tablet report, sending... 09:54:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.957854 31142 ts_manager.cc:98] Registered new tserver with Master: 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:29.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.959578 31386 heartbeater.cc:346] Connected to a master server at 127.30.80.190:60740 09:54:29.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.959852 31386 heartbeater.cc:426] Registering TS with master... 09:54:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.960278 31386 heartbeater.cc:475] Master 127.30.80.190:60740 requested a full tablet report, sending... 09:54:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.969888 31078 ts_manager.cc:98] Registered new tserver with Master: 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:29.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.960916 31388 heartbeater.cc:346] Connected to a master server at 127.30.80.188:49312 09:54:29.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.975643 31388 heartbeater.cc:426] Registering TS with master... 09:54:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.976114 31388 heartbeater.cc:475] Master 127.30.80.188:49312 requested a full tablet report, sending... 09:54:29.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.978250 31207 ts_manager.cc:98] Registered new tserver with Master: 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:29.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:29.980604 31207 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.30.80.129:51344 09:54:30.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:30.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:30.060866 31393 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:30.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:30.061352 31393 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:30.070365 31393 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:30.070817 31393 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.80.130 09:54:30.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.089803 31393 tablet_server_main.cc:78] Tablet server non-default flags: 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/wal 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.130:0 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data/info.pb 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.130 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.80.130 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/logs 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Tablet server version: 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.110033 31393 tablet_server_main.cc:85] Initializing tablet server... 09:54:30.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.112345 31393 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 443027us 09:54:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.112998 31393 fs_manager.cc:263] Metadata directory not provided 09:54:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.113144 31393 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.113687 31393 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.113839 31393 server_base.cc:433] Attempting to create new FS layout instead 09:54:30.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.117300 31393 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data/instance: 09:54:30.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "d5861a09957e486ca7158676ee78f343" 09:54:30.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:30 on quasar-itaafk-1.vpc.cloudera.com" 09:54:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.118340 31393 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/wal/instance: 09:54:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "d5861a09957e486ca7158676ee78f343" 09:54:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:30 on quasar-itaafk-1.vpc.cloudera.com" 09:54:30.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.130935 31393 fs_manager.cc:503] Time spent creating directory manager: real 0.012s user 0.008s sys 0.003s 09:54:30.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.132375 31393 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:30.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.133000 31393 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.140970 31393 fs_manager.cc:419] Time spent opening block manager: real 0.002s user 0.001s sys 0.002s 09:54:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.141114 31393 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/wal 09:54:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "d5861a09957e486ca7158676ee78f343" 09:54:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:30 on quasar-itaafk-1.vpc.cloudera.com" 09:54:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.141528 31393 fs_report.cc:352] FS layout report 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/wal 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/wal 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data/data 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.315135 31393 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.319365 31393 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 09:54:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.321030 31393 tablet_server_main.cc:90] Starting tablet server... 09:54:30.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.357679 31254 catalog_manager.cc:1092] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: acquiring CA information for follower catalog manager: success 09:54:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.361444 31254 catalog_manager.cc:1120] T 00000000000000000000000000000000 P 890d26b4bbaf44d39d7e7107628b0b14: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 09:54:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.464473 31261 catalog_manager.cc:1092] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: acquiring CA information for follower catalog manager: success 09:54:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.475868 31261 catalog_manager.cc:1120] T 00000000000000000000000000000000 P 84cc603334ae465493a2e46b51d40732: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 09:54:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.764963 31393 rpc_server.cc:205] RPC server started. Bound to: 127.30.80.130:56607 09:54:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.767179 31393 webserver.cc:175] Starting webserver on 127.30.80.130:0 09:54:30.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.775951 31393 webserver.cc:186] Document root disabled 09:54:30.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.783881 31393 webserver.cc:313] Webserver started. Bound to: http://127.30.80.130:42590/ 09:54:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.785285 31393 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-1/data/info.pb 09:54:30.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.788386 31042 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 31393 09:54:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.789593 31042 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/wal 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.131:0 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.80.131 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.131 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/logs 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data/info.pb 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false with env {} 09:54:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.883409 31393 tablet_server_main.cc:93] Tablet server successfully started. 09:54:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.906816 31519 heartbeater.cc:346] Connected to a master server at 127.30.80.190:60740 09:54:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.907171 31519 heartbeater.cc:426] Registering TS with master... 09:54:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.907943 31519 heartbeater.cc:475] Master 127.30.80.190:60740 requested a full tablet report, sending... 09:54:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.911442 31078 ts_manager.cc:98] Registered new tserver with Master: d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.908746 31523 heartbeater.cc:346] Connected to a master server at 127.30.80.189:34393 09:54:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.927816 31523 heartbeater.cc:426] Registering TS with master... 09:54:30.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.928350 31523 heartbeater.cc:475] Master 127.30.80.189:34393 requested a full tablet report, sending... 09:54:30.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.931118 31142 ts_manager.cc:98] Registered new tserver with Master: d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.909188 31524 heartbeater.cc:346] Connected to a master server at 127.30.80.188:49312 09:54:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.934624 31524 heartbeater.cc:426] Registering TS with master... 09:54:30.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.935133 31524 heartbeater.cc:475] Master 127.30.80.188:49312 requested a full tablet report, sending... 09:54:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.937189 31207 ts_manager.cc:98] Registered new tserver with Master: d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.939013 31207 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.30.80.130:35148 09:54:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:30.984196 31388 heartbeater.cc:467] Master 127.30.80.188:49312 was elected leader, sending a full tablet report... 09:54:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:31.050246 31520 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:31.051362 31520 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:31.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:31.068522 31520 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:31.069396 31520 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.80.131 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.118016 31520 tablet_server_main.cc:78] Tablet server non-default flags: 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/wal 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.80.131:0 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data/info.pb 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.80.131 09:54:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.80.190:60740,127.30.80.189:34393,127.30.80.188:49312 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.80.131 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/logs 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Tablet server version: 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.147191 31520 tablet_server_main.cc:85] Initializing tablet server... 09:54:31.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.150575 31520 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 443527us 09:54:31.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.151556 31520 fs_manager.cc:263] Metadata directory not provided 09:54:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.151932 31520 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.152477 31520 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.152844 31520 server_base.cc:433] Attempting to create new FS layout instead 09:54:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.158131 31520 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data/instance: 09:54:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "a9eb967bfc62445ab8b89d04df45185d" 09:54:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:31 on quasar-itaafk-1.vpc.cloudera.com" 09:54:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.159585 31520 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/wal/instance: 09:54:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "a9eb967bfc62445ab8b89d04df45185d" 09:54:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:31 on quasar-itaafk-1.vpc.cloudera.com" 09:54:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.194550 31520 fs_manager.cc:503] Time spent creating directory manager: real 0.034s user 0.016s sys 0.012s 09:54:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.196939 31520 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.197827 31520 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.215601 31520 fs_manager.cc:419] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s 09:54:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.216140 31520 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/wal 09:54:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "a9eb967bfc62445ab8b89d04df45185d" 09:54:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:31 on quasar-itaafk-1.vpc.cloudera.com" 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.216773 31520 fs_report.cc:352] FS layout report 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/wal 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/wal 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data/data 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:31.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.483121 31520 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.488543 31520 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 09:54:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.490319 31520 tablet_server_main.cc:90] Starting tablet server... 09:54:31.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.702405 31520 rpc_server.cc:205] RPC server started. Bound to: 127.30.80.131:36267 09:54:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.704632 31520 webserver.cc:175] Starting webserver on 127.30.80.131:0 09:54:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.704936 31520 webserver.cc:186] Document root disabled 09:54:31.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.707157 31520 webserver.cc:313] Webserver started. Bound to: http://127.30.80.131:45014/ 09:54:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.708456 31520 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6729593051901073409/ts-2/data/info.pb 09:54:31.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.709137 31042 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 31520 09:54:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.765151 31520 tablet_server_main.cc:93] Tablet server successfully started. 09:54:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.798426 31660 heartbeater.cc:346] Connected to a master server at 127.30.80.188:49312 09:54:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.798970 31660 heartbeater.cc:426] Registering TS with master... 09:54:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.799744 31660 heartbeater.cc:475] Master 127.30.80.188:49312 requested a full tablet report, sending... 09:54:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.805912 31654 heartbeater.cc:346] Connected to a master server at 127.30.80.190:60740 09:54:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.806165 31654 heartbeater.cc:426] Registering TS with master... 09:54:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.806509 31207 ts_manager.cc:98] Registered new tserver with Master: a9eb967bfc62445ab8b89d04df45185d (127.30.80.131:36267) 09:54:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.806586 31654 heartbeater.cc:475] Master 127.30.80.190:60740 requested a full tablet report, sending... 09:54:31.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.808338 31207 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.30.80.131:40237 09:54:31.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.809041 31078 ts_manager.cc:98] Registered new tserver with Master: a9eb967bfc62445ab8b89d04df45185d (127.30.80.131:36267) 09:54:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.798871 31659 heartbeater.cc:346] Connected to a master server at 127.30.80.189:34393 09:54:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.813823 31659 heartbeater.cc:426] Registering TS with master... 09:54:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.814224 31659 heartbeater.cc:475] Master 127.30.80.189:34393 requested a full tablet report, sending... 09:54:31.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.816709 31142 ts_manager.cc:98] Registered new tserver with Master: a9eb967bfc62445ab8b89d04df45185d (127.30.80.131:36267) 09:54:31.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.819051 31042 external_mini_cluster.cc:535] 3 TS(s) registered with all masters 09:54:31.819 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: 09:54:31.819 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: get_masters { } 09:54:31.821 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.30.80.190" port: 60740 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.30.80.189" port: 34393 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.30.80.188" port: 49312 } } } 09:54:31.821 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: get_tservers { } 09:54:31.823 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.30.80.129" port: 40692 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.30.80.130" port: 56607 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.30.80.131" port: 36267 } } } 09:54:31.823 [INFO - Test worker] (KuduTestHarness.java:139) Creating a new Kudu client... 09:54:31.864 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 09:54:31.874 [DEBUG - New I/O worker #11] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.30.80.189 09:54:31.879 [DEBUG - New I/O worker #10] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.30.80.190 09:54:31.881 [DEBUG - New I/O worker #12] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.30.80.188 09:54:31.897 [DEBUG - New I/O worker #11] (Negotiator.java:764) Authenticated connection [id: 0x269e306f, /127.30.80.189:53023 => /127.30.80.189:34393] using SASL/PLAIN 09:54:31.910 [DEBUG - New I/O worker #12] (Negotiator.java:764) Authenticated connection [id: 0x86158178, /127.30.80.188:47596 => /127.30.80.188:49312] using SASL/PLAIN 09:54:31.910 [DEBUG - New I/O worker #10] (Negotiator.java:764) Authenticated connection [id: 0xef2e9b18, /127.30.80.190:51329 => /127.30.80.190:60740] using SASL/PLAIN 09:54:31.918 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:1986) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 09:54:31.919 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=54), Traces: [0ms] querying master, [4ms] Sub rpc: ConnectToMaster sending RPC to server master-127.30.80.190:60740, [4ms] Sub rpc: ConnectToMaster sending RPC to server master-127.30.80.189:34393, [5ms] Sub rpc: ConnectToMaster sending RPC to server master-127.30.80.188:49312, [35ms] Sub rpc: ConnectToMaster received from server master-127.30.80.189:34393 response OK, [48ms] Sub rpc: ConnectToMaster received from server master-127.30.80.190:60740 response OK, [52ms] Sub rpc: ConnectToMaster received from server master-127.30.80.188:49312 response OK, null) after lookup 09:54:31.920 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=retry RPC@1059768092 returned Deferred@991249441(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after retry RPC@1059768092), errback=(continuation of Deferred@2115489063 after retry RPC@1059768092)), so the following Deferred is getting paused: Deferred@2115489063(state=PAUSED, result=Deferred@991249441, callback=org.apache.kudu.client.AsyncKuduClient$1@2658f213 -> org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3 -> wakeup thread Test worker, errback=passthrough -> passthrough -> wakeup thread Test worker) 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.920904 31207 catalog_manager.cc:1376] Servicing CreateTable request from {username='jenkins'} at 127.30.80.188:47596: 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "org.apache.kudu.client.TestHybridTime-1542822859088" 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) schema { 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) columns { 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "key" 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) type: STRING 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) is_key: true 09:54:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) is_nullable: false 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) cfile_block_size: 0 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) partition_schema { 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) range_schema { 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) columns { 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "key" 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:31.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:31.939095 31207 catalog_manager.cc:1527] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestHybridTime-1542822859088 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 09:54:31.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:31.942677 31524 heartbeater.cc:467] Master 127.30.80.188:49312 was elected leader, sending a full tablet report... 09:54:31.986 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@2658f213@643363347 returned Deferred@1585885802(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@1e383268 -> (continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$1@2658f213@643363347), errback=passthrough -> (continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$1@2658f213@643363347)), so the following Deferred is getting paused: Deferred@2115489063(state=PAUSED, result=Deferred@1585885802, callback=org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3 -> wakeup thread Test worker, errback=passthrough -> wakeup thread Test worker) 09:54:31.989 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:737) Opened table 65675a0af92245549eefea0dabecf2c7 09:54:31.990 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067)), so the following Deferred is getting paused: Deferred@2115489063(state=PAUSED, result=Deferred@2140242902, callback=wakeup thread Test worker, errback=wakeup thread Test worker) 09:54:32.015 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180)), so the following Deferred is getting paused: Deferred@1166154075(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.025766 31323 tablet_service.cc:772] Processing CreateTablet for tablet 90ddcffefb0744dda7efc4a2dcc798a3 (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=65675a0af92245549eefea0dabecf2c7]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.026759 31323 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 90ddcffefb0744dda7efc4a2dcc798a3. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.030591 31591 tablet_service.cc:772] Processing CreateTablet for tablet 90ddcffefb0744dda7efc4a2dcc798a3 (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=65675a0af92245549eefea0dabecf2c7]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:32.033 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305)), so the following Deferred is getting paused: Deferred@1770682202(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.032469 31591 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 90ddcffefb0744dda7efc4a2dcc798a3. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.038902 31455 tablet_service.cc:772] Processing CreateTablet for tablet 90ddcffefb0744dda7efc4a2dcc798a3 (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=65675a0af92245549eefea0dabecf2c7]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.039916 31455 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 90ddcffefb0744dda7efc4a2dcc798a3. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:32.052 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724)), so the following Deferred is getting paused: Deferred@1979369578(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.061609 31455 ts_tablet_manager.cc:1173] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Registered tablet (data state: TABLET_DATA_READY) 09:54:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.067502 31693 ts_tablet_manager.cc:1028] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Bootstrapping tablet 09:54:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.069366 31693 tablet_bootstrap.cc:438] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Bootstrap starting. 09:54:32.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.071239 31323 ts_tablet_manager.cc:1173] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Registered tablet (data state: TABLET_DATA_READY) 09:54:32.071 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284)), so the following Deferred is getting paused: Deferred@1668103093(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.074985 31591 ts_tablet_manager.cc:1173] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Registered tablet (data state: TABLET_DATA_READY) 09:54:32.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.075237 31693 tablet_bootstrap.cc:592] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: No blocks or log segments found. Creating new log. 09:54:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.077217 31693 log.cc:525] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Log is configured to *not* fsync() on all Append() calls 09:54:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.078366 31694 ts_tablet_manager.cc:1028] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Bootstrapping tablet 09:54:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.080112 31694 tablet_bootstrap.cc:438] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Bootstrap starting. 09:54:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.081549 31695 ts_tablet_manager.cc:1028] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Bootstrapping tablet 09:54:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.083338 31695 tablet_bootstrap.cc:438] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Bootstrap starting. 09:54:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.083976 31694 tablet_bootstrap.cc:592] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: No blocks or log segments found. Creating new log. 09:54:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.085815 31694 log.cc:525] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Log is configured to *not* fsync() on all Append() calls 09:54:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.088116 31695 tablet_bootstrap.cc:592] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: No blocks or log segments found. Creating new log. 09:54:32.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.090052 31695 log.cc:525] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Log is configured to *not* fsync() on all Append() calls 09:54:32.092 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457)), so the following Deferred is getting paused: Deferred@1512404164(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.093065 31693 tablet_bootstrap.cc:438] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: No bootstrap required, opened a new log 09:54:32.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.093508 31693 ts_tablet_manager.cc:1045] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Time spent bootstrapping tablet: real 0.024s user 0.008s sys 0.002s 09:54:32.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.102092 31695 tablet_bootstrap.cc:438] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: No bootstrap required, opened a new log 09:54:32.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.102669 31695 ts_tablet_manager.cc:1045] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Time spent bootstrapping tablet: real 0.020s user 0.010s sys 0.006s 09:54:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.107338 31694 tablet_bootstrap.cc:438] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: No bootstrap required, opened a new log 09:54:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.107798 31694 ts_tablet_manager.cc:1045] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Time spent bootstrapping tablet: real 0.028s user 0.005s sys 0.006s 09:54:32.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.123306 31695 raft_consensus.cc:339] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.124491 31695 raft_consensus.cc:365] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.124863 31695 raft_consensus.cc:704] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9eb967bfc62445ab8b89d04df45185d, State: Initialized, Role: FOLLOWER 09:54:32.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.125857 31695 consensus_queue.cc:229] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.122016 31693 raft_consensus.cc:339] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.130368 31693 raft_consensus.cc:365] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.130707 31693 raft_consensus.cc:704] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5861a09957e486ca7158676ee78f343, State: Initialized, Role: FOLLOWER 09:54:32.131 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457) -> (continuation of Deferred@883392646 after org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457) -> (continuation of Deferred@883392646 after org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115)), so the following Deferred is getting paused: Deferred@883392646(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.131527 31693 consensus_queue.cc:229] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.138733 31660 heartbeater.cc:467] Master 127.30.80.188:49312 was elected leader, sending a full tablet report... 09:54:32.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.136814 31694 raft_consensus.cc:339] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.141005 31694 raft_consensus.cc:365] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.141603 31694 raft_consensus.cc:704] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45eaa67cdbdf4cc7b80bac7981ea344d, State: Initialized, Role: FOLLOWER 09:54:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.143033 31694 consensus_queue.cc:229] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.147671 31695 ts_tablet_manager.cc:1073] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Time spent starting tablet: real 0.045s user 0.022s sys 0.007s 09:54:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.152114 31694 ts_tablet_manager.cc:1073] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Time spent starting tablet: real 0.044s user 0.019s sys 0.005s 09:54:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.159837 31693 ts_tablet_manager.cc:1073] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Time spent starting tablet: real 0.066s user 0.023s sys 0.009s 09:54:32.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.182747 31394 tablet.cc:1766] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:32.191 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@374b3b51@927677265 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457) -> (continuation of Deferred@883392646 after org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115) -> (continuation of Deferred@1178204712 after org.apache.kudu.client.AsyncKuduClient$13@374b3b51@927677265), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457) -> (continuation of Deferred@883392646 after org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115) -> (continuation of Deferred@1178204712 after org.apache.kudu.client.AsyncKuduClient$13@374b3b51@927677265)), so the following Deferred is getting paused: Deferred@1178204712(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.200717 31699 raft_consensus.cc:471] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 09:54:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.201107 31699 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Snoozing failure detection for 1.969s (starting election) 09:54:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.201817 31699 raft_consensus.cc:493] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.203452 31699 leader_election.cc:251] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.205060 31699 leader_election.cc:251] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:32.214 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@15d4e020@366272544 returned Deferred@2140242902(state=PENDING, result=null, callback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457) -> (continuation of Deferred@883392646 after org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115) -> (continuation of Deferred@1178204712 after org.apache.kudu.client.AsyncKuduClient$13@374b3b51@927677265) -> (continuation of Deferred@503312812 after org.apache.kudu.client.AsyncKuduClient$13@15d4e020@366272544), errback=(continuation of Deferred@2115489063 after org.apache.kudu.client.AsyncKuduClient$2@4f23d3e3@1327748067) -> (continuation of Deferred@1166154075 after org.apache.kudu.client.AsyncKuduClient$13@2efceb0c@788327180) -> (continuation of Deferred@1770682202 after org.apache.kudu.client.AsyncKuduClient$13@1a76c8f1@443992305) -> (continuation of Deferred@1979369578 after org.apache.kudu.client.AsyncKuduClient$13@20e755ec@552031724) -> (continuation of Deferred@1668103093 after org.apache.kudu.client.AsyncKuduClient$13@5967a764@1499965284) -> (continuation of Deferred@1512404164 after org.apache.kudu.client.AsyncKuduClient$13@79bf4a01@2042579457) -> (continuation of Deferred@883392646 after org.apache.kudu.client.AsyncKuduClient$13@3e1db463@1042134115) -> (continuation of Deferred@1178204712 after org.apache.kudu.client.AsyncKuduClient$13@374b3b51@927677265) -> (continuation of Deferred@503312812 after org.apache.kudu.client.AsyncKuduClient$13@15d4e020@366272544)), so the following Deferred is getting paused: Deferred@503312812(state=PAUSED, result=Deferred@2140242902, callback=, errback=) 09:54:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.213915 31343 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "90ddcffefb0744dda7efc4a2dcc798a3" candidate_uuid: "a9eb967bfc62445ab8b89d04df45185d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" is_pre_election: true 09:54:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.214550 31343 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Snoozing failure detection for 1.944s (vote granted) 09:54:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.214851 31343 raft_consensus.cc:2352] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a9eb967bfc62445ab8b89d04df45185d in term 0. 09:54:32.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.217890 31542 leader_election.cc:387] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 pre-election: Vote granted by peer 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.218123 31542 leader_election.cc:278] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.219353 31699 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Snoozing failure detection for 1.866s (election complete) 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.219612 31699 raft_consensus.cc:2646] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Leader pre-election won for term 1 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.219874 31699 raft_consensus.cc:471] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.220088 31699 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Snoozing failure detection for 1.939s (starting election) 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.220299 31699 raft_consensus.cc:2892] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 0 FOLLOWER]: Advancing to term 1 09:54:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.220255 31476 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "90ddcffefb0744dda7efc4a2dcc798a3" candidate_uuid: "a9eb967bfc62445ab8b89d04df45185d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5861a09957e486ca7158676ee78f343" is_pre_election: true 09:54:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.220911 31476 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Snoozing failure detection for 1.649s (vote granted) 09:54:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.221177 31476 raft_consensus.cc:2352] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a9eb967bfc62445ab8b89d04df45185d in term 0. 09:54:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.222543 31542 leader_election.cc:387] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 pre-election: Vote granted by peer d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.226320 31699 raft_consensus.cc:493] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.227602 31699 leader_election.cc:251] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 election: Requesting vote from peer 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.228251 31699 leader_election.cc:251] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 election: Requesting vote from peer d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.229503 31343 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "90ddcffefb0744dda7efc4a2dcc798a3" candidate_uuid: "a9eb967bfc62445ab8b89d04df45185d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" 09:54:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.229988 31343 raft_consensus.cc:2892] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 0 FOLLOWER]: Advancing to term 1 09:54:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.230274 31343 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d: Snoozing failure detection for 1.810s (vote granted) 09:54:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.230556 31476 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "90ddcffefb0744dda7efc4a2dcc798a3" candidate_uuid: "a9eb967bfc62445ab8b89d04df45185d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5861a09957e486ca7158676ee78f343" 09:54:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.231000 31476 raft_consensus.cc:2892] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 0 FOLLOWER]: Advancing to term 1 09:54:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.231272 31476 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Snoozing failure detection for 2.202s (vote granted) 09:54:32.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.235553 31343 raft_consensus.cc:2352] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a9eb967bfc62445ab8b89d04df45185d in term 1. 09:54:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.235932 31476 raft_consensus.cc:2352] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a9eb967bfc62445ab8b89d04df45185d in term 1. 09:54:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.236958 31542 leader_election.cc:387] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 election: Vote granted by peer 45eaa67cdbdf4cc7b80bac7981ea344d (127.30.80.129:40692) 09:54:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.237378 31542 leader_election.cc:278] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 09:54:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.238046 31699 raft_consensus.cc:2840] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Snoozing failure detection for 2.044s (election complete) 09:54:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.238479 31699 raft_consensus.cc:2646] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 1 FOLLOWER]: Leader election won for term 1 09:54:32.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.238924 31542 leader_election.cc:387] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [CANDIDATE]: Term 1 election: Vote granted by peer d5861a09957e486ca7158676ee78f343 (127.30.80.130:56607) 09:54:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.245299 31699 raft_consensus.cc:666] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [term 1 LEADER]: Becoming Leader. State: Replica: a9eb967bfc62445ab8b89d04df45185d, State: Running, Role: LEADER 09:54:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.245990 31699 consensus_queue.cc:206] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } } 09:54:32.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.256299 31207 catalog_manager.cc:3781] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d reported cstate change: term changed from 0 to 1, leader changed from to a9eb967bfc62445ab8b89d04df45185d (127.30.80.131). New cstate: current_term: 1 leader_uuid: "a9eb967bfc62445ab8b89d04df45185d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a9eb967bfc62445ab8b89d04df45185d" member_type: VOTER last_known_addr { host: "127.30.80.131" port: 36267 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 } health_report { overall_health: UNKNOWN } } } 09:54:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.263872 31663 tablet.cc:1766] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:32.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.378631 31527 tablet.cc:1766] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:32.434 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:1986) Learned about tablet 90ddcffefb0744dda7efc4a2dcc798a3 for table 'org.apache.kudu.client.TestHybridTime-1542822859088' with partition [, ) 09:54:32.435 [DEBUG - New I/O worker #12] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=90ddcffefb0744dda7efc4a2dcc798a3@[45eaa67cdbdf4cc7b80bac7981ea344d(127.30.80.129:40692),a9eb967bfc62445ab8b89d04df45185d(127.30.80.131:36267)[L],d5861a09957e486ca7158676ee78f343(127.30.80.130:56607)]}] 09:54:32.435 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=3), Traces: [0ms] querying master, [0ms] Sub rpc: GetTableLocations sending RPC to server master-127.30.80.188:49312, [2ms] Sub rpc: GetTableLocations received from server master-127.30.80.188:49312 response OK, null) after lookup 09:54:32.436 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=retry RPC@2061605907 returned Deferred@1216788111(state=PENDING, result=null, callback=(continuation of Deferred@714497186 after retry RPC@2061605907), errback=(continuation of Deferred@714497186 after retry RPC@2061605907)), so the following Deferred is getting paused: Deferred@714497186(state=PAUSED, result=Deferred@1216788111, callback=org.apache.kudu.client.AsyncKuduSession$2@5f649649 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@299398b2 -> wakeup thread Time-limited test) 09:54:32.452 [DEBUG - New I/O worker #13] (Negotiator.java:764) Authenticated connection [id: 0xd38c1a71, /127.30.80.131:39984 => /127.30.80.131:36267] using TOKEN/null 09:54:32.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.460307 31699 consensus_queue.cc:745] Safe time advancement without writes is disabled. Snapshot reads on non-leader replicas may stall if there are no writes in progress. 09:54:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.462561 31343 raft_consensus.cc:1180] T 90ddcffefb0744dda7efc4a2dcc798a3 P 45eaa67cdbdf4cc7b80bac7981ea344d [term 1 FOLLOWER]: Refusing update from remote peer a9eb967bfc62445ab8b89d04df45185d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.462636 31476 raft_consensus.cc:1180] T 90ddcffefb0744dda7efc4a2dcc798a3 P d5861a09957e486ca7158676ee78f343 [term 1 FOLLOWER]: Refusing update from remote peer a9eb967bfc62445ab8b89d04df45185d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.464107 31699 consensus_queue.cc:976] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [LEADER]: Connected to new peer: Peer: permanent_uuid: "45eaa67cdbdf4cc7b80bac7981ea344d" member_type: VOTER last_known_addr { host: "127.30.80.129" port: 40692 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.464853 31702 consensus_queue.cc:976] T 90ddcffefb0744dda7efc4a2dcc798a3 P a9eb967bfc62445ab8b89d04df45185d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d5861a09957e486ca7158676ee78f343" member_type: VOTER last_known_addr { host: "127.30.80.130" port: 56607 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:32.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.484071 31706 mvcc.cc:203] Tried to move safe_time back from 6319402485592244224 to 6319402484727885824. Current Snapshot: MvccSnapshot[committed={T|T < 6319402485592244224}] 09:54:32.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.491031 31707 mvcc.cc:203] Tried to move safe_time back from 6319402485592244224 to 6319402484727885824. Current Snapshot: MvccSnapshot[committed={T|T < 6319402485592244224 or (T in {6319402485592244224})}] 09:54:32.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.507884 31708 mvcc.cc:203] Tried to move safe_time back from 6319402485592244224 to 6319402484727885824. Current Snapshot: MvccSnapshot[committed={T|T < 6319402485592244224}] 09:54:32.514 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[0]: Wed Nov 21 09:54:32 PST 2018 Logical value: 0 09:54:32.525 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[1]: Wed Nov 21 09:54:37 PST 2018 Logical value: 1 09:54:32.538 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[2]: Wed Nov 21 09:54:37 PST 2018 Logical value: 3 09:54:32.538 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@1026271440(state=PENDING, result=null, callback=, errback=)} 09:54:32.539 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@39142bad@957623213 returned Deferred@2081858283(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@757985052 after org.apache.kudu.client.AsyncKuduSession$1@39142bad@957623213), errback=passthrough -> (continuation of Deferred@757985052 after org.apache.kudu.client.AsyncKuduSession$1@39142bad@957623213)), so the following Deferred is getting paused: Deferred@757985052(state=PAUSED, result=Deferred@2081858283, callback=, errback=) 09:54:32.549 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[0]: Wed Nov 21 09:54:37 PST 2018 Logical value: 6 09:54:32.549 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@1293991745(state=PENDING, result=null, callback=, errback=)} 09:54:32.550 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@55f1ac0b@1441901579 returned Deferred@473521572(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@1026271440 after org.apache.kudu.client.AsyncKuduSession$1@55f1ac0b@1441901579), errback=passthrough -> (continuation of Deferred@1026271440 after org.apache.kudu.client.AsyncKuduSession$1@55f1ac0b@1441901579)), so the following Deferred is getting paused: Deferred@1026271440(state=PAUSED, result=Deferred@473521572, callback=, errback=) 09:54:32.560 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[1]: Wed Nov 21 09:54:37 PST 2018 Logical value: 7 09:54:32.560 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@1588466736(state=PENDING, result=null, callback=, errback=)} 09:54:32.560 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@16bc7213@381448723 returned Deferred@1324680507(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@1293991745 after org.apache.kudu.client.AsyncKuduSession$1@16bc7213@381448723), errback=passthrough -> (continuation of Deferred@1293991745 after org.apache.kudu.client.AsyncKuduSession$1@16bc7213@381448723)), so the following Deferred is getting paused: Deferred@1293991745(state=PAUSED, result=Deferred@1324680507, callback=, errback=) 09:54:32.571 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[2]: Wed Nov 21 09:54:37 PST 2018 Logical value: 8 09:54:32.579 [DEBUG - New I/O worker #13] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=90ddcffefb0744dda7efc4a2dcc798a3, scannerId=null, scanRequestTimeout=30000) 09:54:32.583 [DEBUG - New I/O worker #13] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 1 rows, more = false, responseScanTimestamp = 6319402506072244224) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=90ddcffefb0744dda7efc4a2dcc798a3, scannerId=null, scanRequestTimeout=30000) 09:54:32.587 [DEBUG - New I/O worker #13] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 3 rows, more = false, responseScanTimestamp = 6319402506072244229) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=90ddcffefb0744dda7efc4a2dcc798a3, scannerId=null, scanRequestTimeout=30000) 09:54:32.588 [DEBUG - New I/O worker #10] (Connection.java:688) [peer master-127.30.80.190:60740(127.30.80.190:60740)] cleaning up while in state READY due to: connection disconnected 09:54:32.589 [DEBUG - New I/O worker #11] (Connection.java:688) [peer master-127.30.80.189:34393(127.30.80.189:34393)] cleaning up while in state READY due to: connection disconnected 09:54:32.589 [DEBUG - New I/O worker #12] (Connection.java:688) [peer master-127.30.80.188:49312(127.30.80.188:49312)] cleaning up while in state READY due to: connection disconnected 09:54:32.590 [DEBUG - Test worker] (AsyncKuduClient.java:2132) Releasing all remaining resources 09:54:32.590 [DEBUG - New I/O worker #13] (Connection.java:688) [peer a9eb967bfc62445ab8b89d04df45185d(127.30.80.131:36267)] cleaning up while in state READY due to: connection disconnected 09:54:32.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.591327 31042 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 31241 09:54:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.613402 31060 connection.cc:512] server connection from 127.30.80.129:35002 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.129:35002): Connection reset by peer (error 104) 09:54:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.613399 31123 connection.cc:512] server connection from 127.30.80.129:40825 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.129:40825): Connection reset by peer (error 104) 09:54:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.614123 31542 connection.cc:512] client connection to 127.30.80.129:40692 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.129:40692): Connection reset by peer (error 104) 09:54:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.614277 31191 connection.cc:512] server connection from 127.30.80.129:51344 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.129:51344): Connection reset by peer (error 104) 09:54:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.615156 31042 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 31393 09:54:32.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.634160 31125 connection.cc:512] server connection from 127.30.80.130:43804 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.130:43804): Connection reset by peer (error 104) 09:54:32.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.634169 31060 connection.cc:512] server connection from 127.30.80.130:51927 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.130:51927): Connection reset by peer (error 104) 09:54:32.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.634984 31042 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 31520 09:54:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.635999 31188 connection.cc:512] server connection from 127.30.80.130:35148 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.130:35148): Connection reset by peer (error 104) 09:54:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.656597 31059 connection.cc:512] server connection from 127.30.80.131:43188 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.131:43188): Connection reset by peer (error 104) 09:54:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.657589 31123 connection.cc:512] server connection from 127.30.80.131:56074 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.131:56074): Connection reset by peer (error 104) 09:54:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:32.657624 31191 connection.cc:512] server connection from 127.30.80.131:40237 recv error: Network error: failed to read from TLS socket (remote: 127.30.80.131:40237): Connection reset by peer (error 104) 09:54:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.658291 31042 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 31050 09:54:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.679790 31042 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 31112 09:54:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:32.700812 31042 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 31176 09:54:33.742 [ERROR - Test worker] (RetryRule.java:76) test(org.apache.kudu.client.TestHybridTime): failed run 2 java.lang.AssertionError: expected:<4> but was:<3> at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.failNotEquals(Assert.java:834) at org.junit.Assert.assertEquals(Assert.java:645) at org.junit.Assert.assertEquals(Assert.java:631) at org.apache.kudu.client.TestHybridTime.test(TestHybridTime.java:167) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 09:54:33.743 [INFO - Test worker] (KuduTestHarness.java:137) Creating a new MiniKuduCluster... 09:54:33.743 [INFO - Test worker] (MiniKuduCluster.java:146) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 09:54:33.744 [INFO - Test worker] (KuduBinaryLocator.java:52) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 09:54:33.744 [INFO - Test worker] (MiniKuduCluster.java:197) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 09:54:33.745 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614" extra_tserver_flags: "--safe_time_advancement_without_writes=false" hms_mode: NONE mini_kdc_options { } } 09:54:33.942 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: 09:54:33.943 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: start_cluster { } 09:54:33.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:33.968422 31717 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:33.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/wal 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=localhost 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.126:55145 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.126 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/logs 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data/info.pb 09:54:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 with env {} 09:54:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.133754 31725 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.134178 31725 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.144215 31725 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 09:54:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.144373 31725 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.144621 31725 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 09:54:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.144798 31725 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 09:54:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.164335 31725 master_main.cc:73] Master server non-default flags: 09:54:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data 09:54:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/wal 09:54:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.126:55145 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data/info.pb 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.126 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/logs 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Master server version: 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.178498 31725 master_main.cc:80] Initializing master server... 09:54:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.181388 31725 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 445027us 09:54:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.182225 31725 fs_manager.cc:263] Metadata directory not provided 09:54:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.182381 31725 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.182816 31725 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:34.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.182943 31725 server_base.cc:433] Attempting to create new FS layout instead 09:54:34.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.186961 31725 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data/instance: 09:54:34.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" 09:54:34.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:34 on quasar-itaafk-1.vpc.cloudera.com" 09:54:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.187845 31725 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/wal/instance: 09:54:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" 09:54:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:34 on quasar-itaafk-1.vpc.cloudera.com" 09:54:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.202023 31725 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.006s sys 0.006s 09:54:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.203372 31725 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:34.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.204689 31725 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.212797 31725 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.212931 31725 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/wal 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:34 on quasar-itaafk-1.vpc.cloudera.com" 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.213280 31725 fs_report.cc:352] FS layout report 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/wal 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/wal 09:54:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data/data 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:34.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.355592 31725 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.358211 31725 master_main.cc:83] Starting Master server... 09:54:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.450493 31725 rpc_server.cc:205] RPC server started. Bound to: 127.30.249.126:55145 09:54:34.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.451596 31725 webserver.cc:175] Starting webserver on 127.30.249.126:0 09:54:34.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.451735 31725 webserver.cc:186] Document root disabled 09:54:34.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.453850 31725 webserver.cc:313] Webserver started. Bound to: http://127.30.249.126:45151/ 09:54:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.454751 31725 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-0/data/info.pb 09:54:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.458855 31786 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:34.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.465401 31717 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 31725 09:54:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.466758 31717 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/wal 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=localhost 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.125:55873 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.125 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/logs 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data/info.pb 09:54:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 with env {} 09:54:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.474776 31786 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } has no permanent_uuid. Determining permanent_uuid... 09:54:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.495719 31786 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } has no permanent_uuid. Determining permanent_uuid... 09:54:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.497601 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.125:55873: Network error: Client connection negotiation failed: client connection to 127.30.249.125:55873: connect: Connection refused (error 111) 09:54:34.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.517041 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } attempt: 1 09:54:34.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.518381 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.125:55873: Network error: Client connection negotiation failed: client connection to 127.30.249.125:55873: connect: Connection refused (error 111) 09:54:34.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.566742 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } attempt: 2 09:54:34.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.568150 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.125:55873: Network error: Client connection negotiation failed: client connection to 127.30.249.125:55873: connect: Connection refused (error 111) 09:54:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.636873 31787 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.637238 31787 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.647130 31787 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 09:54:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.647260 31787 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.647370 31787 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 09:54:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.647533 31787 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.667241 31787 master_main.cc:73] Master server non-default flags: 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/wal 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.125:55873 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data/info.pb 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.125 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/logs 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Master server version: 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.679522 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } attempt: 3 09:54:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.680914 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.125:55873: Network error: Client connection negotiation failed: client connection to 127.30.249.125:55873: connect: Connection refused (error 111) 09:54:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.681213 31787 master_main.cc:80] Initializing master server... 09:54:34.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.683255 31787 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 445027us 09:54:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.684027 31787 fs_manager.cc:263] Metadata directory not provided 09:54:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.684157 31787 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.684406 31787 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.684516 31787 server_base.cc:433] Attempting to create new FS layout instead 09:54:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.688401 31787 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data/instance: 09:54:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "fdf4a0d4f0d9464f986a863286bea4f8" 09:54:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:34 on quasar-itaafk-1.vpc.cloudera.com" 09:54:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.689157 31787 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/wal/instance: 09:54:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "fdf4a0d4f0d9464f986a863286bea4f8" 09:54:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:34 on quasar-itaafk-1.vpc.cloudera.com" 09:54:34.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.703274 31787 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.008s sys 0.005s 09:54:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.704519 31787 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.705787 31787 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.713923 31787 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.714054 31787 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/wal 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "fdf4a0d4f0d9464f986a863286bea4f8" 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:34 on quasar-itaafk-1.vpc.cloudera.com" 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.714324 31787 fs_report.cc:352] FS layout report 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/wal 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/wal 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data/data 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.857247 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } attempt: 4 09:54:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:34.858700 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.125:55873: Network error: Client connection negotiation failed: client connection to 127.30.249.125:55873: connect: Connection refused (error 111) 09:54:34.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.893306 31787 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.895918 31787 master_main.cc:83] Starting Master server... 09:54:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.986171 31787 rpc_server.cc:205] RPC server started. Bound to: 127.30.249.125:55873 09:54:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.987349 31787 webserver.cc:175] Starting webserver on 127.30.249.125:0 09:54:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.987463 31787 webserver.cc:186] Document root disabled 09:54:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.989470 31787 webserver.cc:313] Webserver started. Bound to: http://127.30.249.125:58741/ 09:54:34.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.990339 31787 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-1/data/info.pb 09:54:34.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.993760 31717 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 31787 09:54:34.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.994659 31850 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:34.995087 31717 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/wal 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=localhost 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.124:57750 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.124 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/logs 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data/info.pb 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 with env {} 09:54:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.010063 31850 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.029145 31850 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.042237 31850 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.044085 31850 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.063572 31850 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 1 09:54:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.065126 31850 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.113473 31850 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 2 09:54:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.114846 31850 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.162055 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } attempt: 5 09:54:35.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.169005 31786 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.169831 31851 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.170244 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.170534 31851 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.180533 31851 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 09:54:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.180662 31851 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.180827 31851 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 09:54:35.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.181092 31851 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 09:54:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.200479 31851 master_main.cc:73] Master server non-default flags: 09:54:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data 09:54:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/wal 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_ca_key_size=1024 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --master_addresses=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tsk_num_rsa_bits=512 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.124:57750 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_reuseport=true 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data/info.pb 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.124 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/logs 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Master server version: 09:54:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.208601 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 1 09:54:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.209942 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.214903 31851 master_main.cc:80] Initializing master server... 09:54:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.216991 31851 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 445527us 09:54:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.217838 31851 fs_manager.cc:263] Metadata directory not provided 09:54:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.217993 31851 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.218341 31851 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.218472 31851 server_base.cc:433] Attempting to create new FS layout instead 09:54:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.222486 31851 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data/instance: 09:54:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "5a9833f5a8b74971adbaf2731a6d5034" 09:54:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:35 on quasar-itaafk-1.vpc.cloudera.com" 09:54:35.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.223342 31851 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/wal/instance: 09:54:35.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "5a9833f5a8b74971adbaf2731a6d5034" 09:54:35.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:35 on quasar-itaafk-1.vpc.cloudera.com" 09:54:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.226208 31850 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 3 09:54:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.227494 31850 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.238260 31851 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.009s sys 0.004s 09:54:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.239599 31851 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.240948 31851 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.249126 31851 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s 09:54:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.249263 31851 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/wal 09:54:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "5a9833f5a8b74971adbaf2731a6d5034" 09:54:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:35 on quasar-itaafk-1.vpc.cloudera.com" 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.249598 31851 fs_report.cc:352] FS layout report 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/wal 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/wal 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data/data 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.258291 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 2 09:54:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.259817 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.360662 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 3 09:54:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.362177 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.403785 31850 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 4 09:54:35.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.405185 31850 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.429760 31851 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.432368 31851 master_main.cc:83] Starting Master server... 09:54:35.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.502604 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 4 09:54:35.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.504318 31786 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.30.249.124:57750: Network error: Client connection negotiation failed: client connection to 127.30.249.124:57750: connect: Connection refused (error 111) 09:54:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.523120 31851 rpc_server.cc:205] RPC server started. Bound to: 127.30.249.124:57750 09:54:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.524046 31851 webserver.cc:175] Starting webserver on 127.30.249.124:0 09:54:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.524149 31851 webserver.cc:186] Document root disabled 09:54:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.526034 31851 webserver.cc:313] Webserver started. Bound to: http://127.30.249.124:51845/ 09:54:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.526857 31851 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/master-2/data/info.pb 09:54:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.531198 31915 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.533501 31717 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 31851 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.534978 31717 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/wal 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.65:0 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.249.65 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.65 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/logs 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data/info.pb 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false with env {} 09:54:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.547938 31915 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.566558 31915 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.573940 31915 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } has no permanent_uuid. Determining permanent_uuid... 09:54:35.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.589581 31915 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Bootstrap starting. 09:54:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.592339 31915 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: No blocks or log segments found. Creating new log. 09:54:35.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.593503 31915 log.cc:525] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Log is configured to *not* fsync() on all Append() calls 09:54:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.598551 31915 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: No bootstrap required, opened a new log 09:54:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.609920 31915 raft_consensus.cc:339] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.610425 31915 raft_consensus.cc:365] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.610620 31915 raft_consensus.cc:704] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a9833f5a8b74971adbaf2731a6d5034, State: Initialized, Role: FOLLOWER 09:54:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.611380 31915 consensus_queue.cc:229] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.614375 31915 sys_catalog.cc:337] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.615320 31915 sys_catalog.cc:340] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: This master's current role is: FOLLOWER 09:54:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.616992 31915 sys_catalog.cc:424] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: configured and running, proceeding with master startup. 09:54:35.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.615734 31921 sys_catalog.cc:337] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.685855 31871 tablet.cc:1766] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.685225 31921 sys_catalog.cc:340] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: This master's current role is: FOLLOWER 09:54:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.688659 31851 master_main.cc:86] Master server successfully started. 09:54:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.693964 31922 catalog_manager.cc:1094] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: acquiring CA information for follower catalog manager: Not found: root CA entry not found 09:54:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.694226 31922 catalog_manager.cc:584] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 09:54:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.708528 31850 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 5 09:54:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.714733 31916 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.715528 31916 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:35.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.724701 31850 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: Bootstrap starting. 09:54:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.725205 31916 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.725574 31916 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.249.65 09:54:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.729030 31850 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: No blocks or log segments found. Creating new log. 09:54:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.731026 31850 log.cc:525] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: Log is configured to *not* fsync() on all Append() calls 09:54:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.736133 31850 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: No bootstrap required, opened a new log 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.747782 31916 tablet_server_main.cc:78] Tablet server non-default flags: 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/wal 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.65:0 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data/info.pb 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.65 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.249.65 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/logs 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Tablet server version: 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.756253 31850 raft_consensus.cc:339] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.757028 31850 raft_consensus.cc:365] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.757242 31850 raft_consensus.cc:704] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fdf4a0d4f0d9464f986a863286bea4f8, State: Initialized, Role: FOLLOWER 09:54:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.758095 31850 consensus_queue.cc:229] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.761067 31850 sys_catalog.cc:337] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.761976 31850 sys_catalog.cc:340] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: This master's current role is: FOLLOWER 09:54:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.763499 31850 sys_catalog.cc:424] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: configured and running, proceeding with master startup. 09:54:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.769682 31786 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } attempt: 5 09:54:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.770283 31916 tablet_server_main.cc:85] Initializing tablet server... 09:54:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.774713 31916 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 445527us 09:54:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.775620 31916 fs_manager.cc:263] Metadata directory not provided 09:54:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.775985 31916 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.776501 31916 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.776841 31916 server_base.cc:433] Attempting to create new FS layout instead 09:54:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.783753 31916 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data/instance: 09:54:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "80ad009ef0e8437ba853f76a4c6113ec" 09:54:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:35 on quasar-itaafk-1.vpc.cloudera.com" 09:54:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.785306 31916 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/wal/instance: 09:54:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "80ad009ef0e8437ba853f76a4c6113ec" 09:54:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:35 on quasar-itaafk-1.vpc.cloudera.com" 09:54:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.796398 31786 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: Bootstrap starting. 09:54:35.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.800463 31786 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: No blocks or log segments found. Creating new log. 09:54:35.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.806654 31786 log.cc:525] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: Log is configured to *not* fsync() on all Append() calls 09:54:35.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.817064 31916 fs_manager.cc:503] Time spent creating directory manager: real 0.031s user 0.014s sys 0.011s 09:54:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.818891 31916 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.820812 31916 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:35.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.821920 31786 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: No bootstrap required, opened a new log 09:54:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.838727 31916 fs_manager.cc:419] Time spent opening block manager: real 0.007s user 0.001s sys 0.005s 09:54:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.839231 31916 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/wal 09:54:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "80ad009ef0e8437ba853f76a4c6113ec" 09:54:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:35 on quasar-itaafk-1.vpc.cloudera.com" 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.839920 31916 fs_report.cc:352] FS layout report 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/wal 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/wal 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data/data 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:35.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.761420 31929 sys_catalog.cc:337] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:35.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.863240 31929 sys_catalog.cc:340] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: This master's current role is: FOLLOWER 09:54:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.875511 31787 master_main.cc:86] Master server successfully started. 09:54:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.870479 31786 raft_consensus.cc:339] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.884721 31786 raft_consensus.cc:365] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:35.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.885219 31786 raft_consensus.cc:704] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c2ea9930d4a45f1a96ecb19ad06cd30, State: Initialized, Role: FOLLOWER 09:54:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.886531 31930 catalog_manager.cc:1094] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: acquiring CA information for follower catalog manager: Not found: root CA entry not found 09:54:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.886196 31786 consensus_queue.cc:229] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.886917 31930 catalog_manager.cc:584] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 09:54:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.893507 31939 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.894479 31939 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: This master's current role is: FOLLOWER 09:54:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.899022 31786 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.899634 31786 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: This master's current role is: FOLLOWER 09:54:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.900354 31806 tablet.cc:1766] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.900825 31786 sys_catalog.cc:424] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: configured and running, proceeding with master startup. 09:54:35.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.920404 31725 master_main.cc:86] Master server successfully started. 09:54:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.980646 31921 raft_consensus.cc:471] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 09:54:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.981005 31921 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Snoozing failure detection for 1.842s (starting election) 09:54:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.981328 31921 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.983196 31921 leader_election.cc:251] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 4c2ea9930d4a45f1a96ecb19ad06cd30 (127.30.249.126:55145) 09:54:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.983933 31921 leader_election.cc:251] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer fdf4a0d4f0d9464f986a863286bea4f8 (127.30.249.125:55873) 09:54:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.985497 31827 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5a9833f5a8b74971adbaf2731a6d5034" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" is_pre_election: true 09:54:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.986196 31827 raft_consensus.cc:2840] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: Snoozing failure detection for 2.106s (vote granted) 09:54:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.986486 31827 raft_consensus.cc:2352] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5a9833f5a8b74971adbaf2731a6d5034 in term 0. 09:54:35.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.987639 31862 leader_election.cc:387] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 pre-election: Vote granted by peer fdf4a0d4f0d9464f986a863286bea4f8 (127.30.249.125:55873) 09:54:35.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.987932 31862 leader_election.cc:278] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 09:54:35.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.988513 31921 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Snoozing failure detection for 1.656s (election complete) 09:54:35.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.988755 31921 raft_consensus.cc:2646] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Leader pre-election won for term 1 09:54:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.989028 31921 raft_consensus.cc:471] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 09:54:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.989248 31921 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Snoozing failure detection for 2.198s (starting election) 09:54:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.989476 31921 raft_consensus.cc:2892] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 0 FOLLOWER]: Advancing to term 1 09:54:35.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.994736 31921 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:35.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.998865 31921 leader_election.cc:251] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 election: Requesting vote from peer 4c2ea9930d4a45f1a96ecb19ad06cd30 (127.30.249.126:55145) 09:54:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.006388 31921 leader_election.cc:251] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 election: Requesting vote from peer fdf4a0d4f0d9464f986a863286bea4f8 (127.30.249.125:55873) 09:54:36.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.007810 31827 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5a9833f5a8b74971adbaf2731a6d5034" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" 09:54:36.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.008293 31827 raft_consensus.cc:2892] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 0 FOLLOWER]: Advancing to term 1 09:54:36.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.008615 31827 raft_consensus.cc:2840] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: Snoozing failure detection for 2.172s (vote granted) 09:54:36.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.015475 31827 raft_consensus.cc:2352] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5a9833f5a8b74971adbaf2731a6d5034 in term 1. 09:54:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.016278 31862 leader_election.cc:387] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 election: Vote granted by peer fdf4a0d4f0d9464f986a863286bea4f8 (127.30.249.125:55873) 09:54:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.016549 31862 leader_election.cc:278] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 09:54:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.017084 31921 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Snoozing failure detection for 2.168s (election complete) 09:54:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.017341 31921 raft_consensus.cc:2646] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 1 FOLLOWER]: Leader election won for term 1 09:54:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.022658 31921 raft_consensus.cc:666] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 1 LEADER]: Becoming Leader. State: Replica: 5a9833f5a8b74971adbaf2731a6d5034, State: Running, Role: LEADER 09:54:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.027889 31921 consensus_queue.cc:206] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:35.925740 31941 catalog_manager.cc:1094] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: acquiring CA information for follower catalog manager: Not found: root CA entry not found 09:54:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:36.040006 31941 catalog_manager.cc:584] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 09:54:36.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.045224 31943 sys_catalog.cc:337] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5a9833f5a8b74971adbaf2731a6d5034. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.045960 31943 sys_catalog.cc:340] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: This master's current role is: LEADER 09:54:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.057374 31945 catalog_manager.cc:1036] Loading table and tablet metadata into memory... 09:54:36.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.061048 31945 catalog_manager.cc:1047] Initializing Kudu internal certificate authority... 09:54:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:36.112506 31742 tablet.cc:1766] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:35.985270 31763 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5a9833f5a8b74971adbaf2731a6d5034" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" is_pre_election: true 09:54:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.007491 31762 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5a9833f5a8b74971adbaf2731a6d5034" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" 09:54:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.125483 31916 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:36.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.134496 31916 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 09:54:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.141153 31916 tablet_server_main.cc:90] Starting tablet server... 09:54:36.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.207135 31827 raft_consensus.cc:1180] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 1 FOLLOWER]: Refusing update from remote peer 5a9833f5a8b74971adbaf2731a6d5034: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.208978 31943 consensus_queue.cc:976] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.237658 31929 sys_catalog.cc:337] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5a9833f5a8b74971adbaf2731a6d5034. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.238356 31929 sys_catalog.cc:340] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: This master's current role is: FOLLOWER 09:54:36.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.245121 31958 mvcc.cc:203] Tried to move safe_time back from 6319402500916154368 to 6319402500218925056. Current Snapshot: MvccSnapshot[committed={T|T < 6319402500916154368}] 09:54:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.253669 31943 sys_catalog.cc:337] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.254420 31943 sys_catalog.cc:340] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: This master's current role is: LEADER 09:54:36.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.255447 31929 sys_catalog.cc:337] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.256112 31929 sys_catalog.cc:340] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [sys.catalog]: This master's current role is: FOLLOWER 09:54:36.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.257961 31945 catalog_manager.cc:931] Generated new certificate authority record 09:54:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.260309 31945 catalog_manager.cc:1056] Loading token signing keys... 09:54:36.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.027343 31939 raft_consensus.cc:471] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 09:54:36.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.279352 31939 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: Snoozing failure detection for 2.035s (starting election) 09:54:36.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.279706 31939 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } 09:54:36.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.281424 31939 leader_election.cc:251] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer fdf4a0d4f0d9464f986a863286bea4f8 (127.30.249.125:55873) 09:54:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.282119 31939 leader_election.cc:251] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 5a9833f5a8b74971adbaf2731a6d5034 (127.30.249.124:57750) 09:54:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.282690 31763 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: Snoozing failure detection for 2.093s (vote granted) 09:54:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.283031 31763 raft_consensus.cc:2352] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5a9833f5a8b74971adbaf2731a6d5034 in term 0. 09:54:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.283041 31826 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" is_pre_election: true 09:54:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.283499 31762 raft_consensus.cc:2327] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Leader election vote request: Denying vote to candidate 5a9833f5a8b74971adbaf2731a6d5034 for term 1 because replica is already servicing an update from a current leader or another vote. 09:54:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.283643 31826 raft_consensus.cc:2313] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 4c2ea9930d4a45f1a96ecb19ad06cd30 for term 1 because replica is either leader or believes a valid leader to be alive. 09:54:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.284588 31733 leader_election.cc:401] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [CANDIDATE]: Term 1 pre-election: Vote denied by peer fdf4a0d4f0d9464f986a863286bea4f8 (127.30.249.125:55873). Message: Invalid argument: T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 4c2ea9930d4a45f1a96ecb19ad06cd30 for term 1 because replica is either leader or believes a valid leader to be alive. 09:54:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.284679 31892 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a9833f5a8b74971adbaf2731a6d5034" is_pre_election: true 09:54:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.285444 31892 raft_consensus.cc:2313] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 1 LEADER]: Leader pre-election vote request: Denying vote to candidate 4c2ea9930d4a45f1a96ecb19ad06cd30 for term 1 because replica is either leader or believes a valid leader to be alive. 09:54:36.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.286974 31735 leader_election.cc:401] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [CANDIDATE]: Term 1 pre-election: Vote denied by peer 5a9833f5a8b74971adbaf2731a6d5034 (127.30.249.124:57750). Message: Invalid argument: T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [term 1 LEADER]: Leader pre-election vote request: Denying vote to candidate 4c2ea9930d4a45f1a96ecb19ad06cd30 for term 1 because replica is either leader or believes a valid leader to be alive. 09:54:36.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.287257 31735 leader_election.cc:278] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. 09:54:36.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.288002 31761 raft_consensus.cc:2892] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Advancing to term 1 09:54:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.289283 31866 leader_election.cc:401] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 election: Vote denied by peer 4c2ea9930d4a45f1a96ecb19ad06cd30 (127.30.249.126:55145). Message: Service unavailable: T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 0 FOLLOWER]: Leader election vote request: Denying vote to candidate 5a9833f5a8b74971adbaf2731a6d5034 for term 1 because replica is already servicing an update from a current leader or another vote. 09:54:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.289903 31866 leader_election.cc:387] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [CANDIDATE]: Term 1 pre-election: Vote granted by peer 4c2ea9930d4a45f1a96ecb19ad06cd30 (127.30.249.126:55145) 09:54:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.293797 31761 raft_consensus.cc:1180] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 1 FOLLOWER]: Refusing update from remote peer 5a9833f5a8b74971adbaf2731a6d5034: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:36.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.295007 31939 raft_consensus.cc:2840] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: Snoozing failure detection for 2.078s (election complete) 09:54:36.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.295248 31939 raft_consensus.cc:2594] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority 09:54:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.296746 31921 consensus_queue.cc:976] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.302377 31945 catalog_manager.cc:4085] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034: Generated new TSK 0 09:54:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.360677 31979 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.361745 31979 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: This master's current role is: FOLLOWER 09:54:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.361989 31943 sys_catalog.cc:337] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.362653 31943 sys_catalog.cc:340] T 00000000000000000000000000000000 P 5a9833f5a8b74971adbaf2731a6d5034 [sys.catalog]: This master's current role is: LEADER 09:54:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.360360 31939 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5a9833f5a8b74971adbaf2731a6d5034. Latest consensus state: current_term: 1 leader_uuid: "5a9833f5a8b74971adbaf2731a6d5034" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c2ea9930d4a45f1a96ecb19ad06cd30" member_type: VOTER last_known_addr { host: "127.30.249.126" port: 55145 } } peers { permanent_uuid: "fdf4a0d4f0d9464f986a863286bea4f8" member_type: VOTER last_known_addr { host: "127.30.249.125" port: 55873 } } peers { permanent_uuid: "5a9833f5a8b74971adbaf2731a6d5034" member_type: VOTER last_known_addr { host: "127.30.249.124" port: 57750 } } } 09:54:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.363238 31939 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30 [sys.catalog]: This master's current role is: FOLLOWER 09:54:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.738880 31916 rpc_server.cc:205] RPC server started. Bound to: 127.30.249.65:59307 09:54:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.740824 31916 webserver.cc:175] Starting webserver on 127.30.249.65:0 09:54:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.741185 31916 webserver.cc:186] Document root disabled 09:54:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.747339 31916 webserver.cc:313] Webserver started. Bound to: http://127.30.249.65:46322/ 09:54:36.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.748612 31916 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-0/data/info.pb 09:54:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.758740 31717 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 31916 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.760145 31717 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/wal 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.66:0 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.249.66 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.66 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:36.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/logs 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data/info.pb 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false with env {} 09:54:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.809595 31916 tablet_server_main.cc:93] Tablet server successfully started. 09:54:36.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.825314 32069 heartbeater.cc:346] Connected to a master server at 127.30.249.126:55145 09:54:36.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.825641 32069 heartbeater.cc:426] Registering TS with master... 09:54:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.827154 32069 heartbeater.cc:475] Master 127.30.249.126:55145 requested a full tablet report, sending... 09:54:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.832036 31753 ts_manager.cc:98] Registered new tserver with Master: 80ad009ef0e8437ba853f76a4c6113ec (127.30.249.65:59307) 09:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.834161 32070 heartbeater.cc:346] Connected to a master server at 127.30.249.125:55873 09:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.834393 32070 heartbeater.cc:426] Registering TS with master... 09:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.834869 32070 heartbeater.cc:475] Master 127.30.249.125:55873 requested a full tablet report, sending... 09:54:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.837911 31817 ts_manager.cc:98] Registered new tserver with Master: 80ad009ef0e8437ba853f76a4c6113ec (127.30.249.65:59307) 09:54:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.827179 32074 heartbeater.cc:346] Connected to a master server at 127.30.249.124:57750 09:54:36.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.849033 32074 heartbeater.cc:426] Registering TS with master... 09:54:36.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.849534 32074 heartbeater.cc:475] Master 127.30.249.124:57750 requested a full tablet report, sending... 09:54:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.852131 31882 ts_manager.cc:98] Registered new tserver with Master: 80ad009ef0e8437ba853f76a4c6113ec (127.30.249.65:59307) 09:54:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.855093 31882 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.30.249.65:38364 09:54:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.893452 31930 catalog_manager.cc:1092] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: acquiring CA information for follower catalog manager: success 09:54:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:36.901691 31930 catalog_manager.cc:1120] T 00000000000000000000000000000000 P fdf4a0d4f0d9464f986a863286bea4f8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 09:54:37.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:37.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:37.005523 32073 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:37.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:37.006768 32073 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:37.023627 32073 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:37.024580 32073 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.249.66 09:54:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.046640 31941 catalog_manager.cc:1092] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: acquiring CA information for follower catalog manager: success 09:54:37.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.049913 31941 catalog_manager.cc:1120] T 00000000000000000000000000000000 P 4c2ea9930d4a45f1a96ecb19ad06cd30: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.066026 32073 tablet_server_main.cc:78] Tablet server non-default flags: 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/wal 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.66:0 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data/info.pb 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.66 09:54:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.249.66 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/logs 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Tablet server version: 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.095069 32073 tablet_server_main.cc:85] Initializing tablet server... 09:54:37.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.098695 32073 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 446527us 09:54:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.099699 32073 fs_manager.cc:263] Metadata directory not provided 09:54:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.100111 32073 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.100689 32073 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:37.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.101094 32073 server_base.cc:433] Attempting to create new FS layout instead 09:54:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.106317 32073 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data/instance: 09:54:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "0717ab552dd7445b87e5ca48cb248966" 09:54:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:37 on quasar-itaafk-1.vpc.cloudera.com" 09:54:37.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.107836 32073 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/wal/instance: 09:54:37.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "0717ab552dd7445b87e5ca48cb248966" 09:54:37.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:37 on quasar-itaafk-1.vpc.cloudera.com" 09:54:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.136391 32073 fs_manager.cc:503] Time spent creating directory manager: real 0.028s user 0.015s sys 0.009s 09:54:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.138718 32073 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:37.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.139624 32073 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:37.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.166420 32073 fs_manager.cc:419] Time spent opening block manager: real 0.016s user 0.001s sys 0.013s 09:54:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.167171 32073 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/wal 09:54:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "0717ab552dd7445b87e5ca48cb248966" 09:54:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:37 on quasar-itaafk-1.vpc.cloudera.com" 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.167894 32073 fs_report.cc:352] FS layout report 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/wal 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/wal 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data/data 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.475570 32073 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:37.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.485285 32073 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 09:54:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.491756 32073 tablet_server_main.cc:90] Starting tablet server... 09:54:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:37.858889 32074 heartbeater.cc:467] Master 127.30.249.124:57750 was elected leader, sending a full tablet report... 09:54:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.050554 32073 rpc_server.cc:205] RPC server started. Bound to: 127.30.249.66:54184 09:54:38.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.052183 32073 webserver.cc:175] Starting webserver on 127.30.249.66:0 09:54:38.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.052318 32073 webserver.cc:186] Document root disabled 09:54:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.054232 32073 webserver.cc:313] Webserver started. Bound to: http://127.30.249.66:47972/ 09:54:38.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.055222 32073 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-1/data/info.pb 09:54:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.060066 31717 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 32073 09:54:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.061391 31717 external_mini_cluster.cc:845] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 09:54:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/wal 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --block_manager=log 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.67:0 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.249.67 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.67 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/logs 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data/info.pb 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags 09:54:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false with env {} 09:54:38.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.105626 32073 tablet_server_main.cc:93] Tablet server successfully started. 09:54:38.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.156226 32202 heartbeater.cc:346] Connected to a master server at 127.30.249.125:55873 09:54:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.156813 32202 heartbeater.cc:426] Registering TS with master... 09:54:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.158052 32202 heartbeater.cc:475] Master 127.30.249.125:55873 requested a full tablet report, sending... 09:54:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.161304 31817 ts_manager.cc:98] Registered new tserver with Master: 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.157369 32208 heartbeater.cc:346] Connected to a master server at 127.30.249.124:57750 09:54:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.168658 32208 heartbeater.cc:426] Registering TS with master... 09:54:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.169152 32208 heartbeater.cc:475] Master 127.30.249.124:57750 requested a full tablet report, sending... 09:54:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.171104 31882 ts_manager.cc:98] Registered new tserver with Master: 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.172927 31882 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.30.249.66:52533 09:54:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.158462 32201 heartbeater.cc:346] Connected to a master server at 127.30.249.126:55145 09:54:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.177408 32201 heartbeater.cc:426] Registering TS with master... 09:54:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.177789 32201 heartbeater.cc:475] Master 127.30.249.126:55145 requested a full tablet report, sending... 09:54:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.180284 31753 ts_manager.cc:98] Registered new tserver with Master: 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) WARNING: Logging before InitGoogleLogging() is written to STDERR 09:54:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:38.274401 32205 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 09:54:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:38.275029 32205 flags.cc:406] Enabled unsafe flag: --never_fsync=true 09:54:38.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:38.292763 32205 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 09:54:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:38.293534 32205 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.249.67 09:54:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.317457 32205 tablet_server_main.cc:78] Tablet server non-default flags: 09:54:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --safe_time_advancement_without_writes=false 09:54:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data 09:54:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/wal 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --ipki_server_key_size=1024 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_bind_addresses=127.30.249.67:0 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --rpc_server_allow_ephemeral_ports=true 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --metrics_log_interval_ms=1000 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_format=pb 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data/info.pb 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_interface=127.30.249.67 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --webserver_port=0 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --tserver_master_addrs=127.30.249.126:55145,127.30.249.125:55873,127.30.249.124:57750 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --never_fsync=true 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --redact=none 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_experimental_flags=true 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --unlock_unsafe_flags=true 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --enable_minidumps=false 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --local_ip_for_outbound_sockets=127.30.249.67 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/logs 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logbuflevel=-1 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) --logtostderr=true 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Tablet server version: 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) kudu 1.9.0-SNAPSHOT 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) revision 66b06de7b14621f821141ac7f9ec4ba96cad5af1 09:54:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build type FASTDEBUG 09:54:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) built by jenkins at 21 Nov 2018 07:24:42 PST on quasar-itaafk-1.vpc.cloudera.com 09:54:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) build id 201982 09:54:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) TSAN enabled 09:54:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.334313 32205 tablet_server_main.cc:85] Initializing tablet server... 09:54:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.336436 32205 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 447027us 09:54:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.337011 32205 fs_manager.cc:263] Metadata directory not provided 09:54:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.337153 32205 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 09:54:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.337476 32205 server_base.cc:432] Could not load existing FS layout: Not found: could not find a healthy instance file 09:54:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.337594 32205 server_base.cc:433] Attempting to create new FS layout instead 09:54:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.340777 32205 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data/instance: 09:54:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "48d79176988849e0a1b5277b50c78b2a" 09:54:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:38 on quasar-itaafk-1.vpc.cloudera.com" 09:54:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.341601 32205 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/wal/instance: 09:54:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "48d79176988849e0a1b5277b50c78b2a" 09:54:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:38 on quasar-itaafk-1.vpc.cloudera.com" 09:54:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.355316 32205 fs_manager.cc:503] Time spent creating directory manager: real 0.013s user 0.008s sys 0.004s 09:54:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.356679 32205 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 09:54:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.357272 32205 file_cache.cc:466] Constructed file cache lbm with capacity 13107 09:54:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.365192 32205 fs_manager.cc:419] Time spent opening block manager: real 0.002s user 0.001s sys 0.001s 09:54:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.365326 32205 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/wal 09:54:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) uuid: "48d79176988849e0a1b5277b50c78b2a" 09:54:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) format_stamp: "Formatted at 2018-11-21 17:54:38 on quasar-itaafk-1.vpc.cloudera.com" 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.365700 32205 fs_report.cc:352] FS layout report 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) -------------------- 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/wal 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/wal 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data/data 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live blocks: 0 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes: 0 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total live bytes (after alignment): 0 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total number of LBM containers: 0 (0 full) 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for missing blocks 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Did not check for orphaned blocks 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM containers with extra space: 0 (0 repaired) 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total full LBM container extra space in bytes: 0 (0 repaired) 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total incomplete LBM containers: 0 (0 repaired) 09:54:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) Total LBM partial records: 0 (0 repaired) 09:54:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.560187 32205 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 09:54:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.563968 32205 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 09:54:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.565238 32205 tablet_server_main.cc:90] Starting tablet server... 09:54:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.674872 32205 rpc_server.cc:205] RPC server started. Bound to: 127.30.249.67:53351 09:54:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.676170 32205 webserver.cc:175] Starting webserver on 127.30.249.67:0 09:54:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.676301 32205 webserver.cc:186] Document root disabled 09:54:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.678203 32205 webserver.cc:313] Webserver started. Bound to: http://127.30.249.67:40493/ 09:54:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.679165 32205 server_base.cc:607] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster4920845319969265614/ts-2/data/info.pb 09:54:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.683454 31717 external_mini_cluster.cc:907] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 32205 09:54:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.709264 32205 tablet_server_main.cc:93] Tablet server successfully started. 09:54:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.751844 32334 heartbeater.cc:346] Connected to a master server at 127.30.249.124:57750 09:54:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.752331 32334 heartbeater.cc:426] Registering TS with master... 09:54:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.752979 32334 heartbeater.cc:475] Master 127.30.249.124:57750 requested a full tablet report, sending... 09:54:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.755367 31881 ts_manager.cc:98] Registered new tserver with Master: 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.756623 31881 master_service.cc:253] Signed X509 certificate for tserver {username='jenkins'} at 127.30.249.67:33354 09:54:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.754025 32333 heartbeater.cc:346] Connected to a master server at 127.30.249.125:55873 09:54:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.765009 32333 heartbeater.cc:426] Registering TS with master... 09:54:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.765331 32333 heartbeater.cc:475] Master 127.30.249.125:55873 requested a full tablet report, sending... 09:54:38.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.767701 31817 ts_manager.cc:98] Registered new tserver with Master: 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.754392 32332 heartbeater.cc:346] Connected to a master server at 127.30.249.126:55145 09:54:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.769152 32332 heartbeater.cc:426] Registering TS with master... 09:54:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.769438 32332 heartbeater.cc:475] Master 127.30.249.126:55145 requested a full tablet report, sending... 09:54:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.771056 31753 ts_manager.cc:98] Registered new tserver with Master: 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.774139 31717 external_mini_cluster.cc:535] 3 TS(s) registered with all masters 09:54:38.774 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: 09:54:38.774 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: get_masters { } 09:54:38.776 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.30.249.126" port: 55145 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.30.249.125" port: 55873 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.30.249.124" port: 57750 } } } 09:54:38.776 [DEBUG - Test worker] (MiniKuduCluster.java:165) Request: get_tservers { } 09:54:38.781 [DEBUG - Test worker] (MiniKuduCluster.java:175) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.30.249.65" port: 59307 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.30.249.66" port: 54184 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.30.249.67" port: 53351 } } } 09:54:38.781 [INFO - Test worker] (KuduTestHarness.java:139) Creating a new Kudu client... 09:54:38.805 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 09:54:38.811 [DEBUG - New I/O worker #19] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.30.249.125 09:54:38.811 [DEBUG - New I/O worker #21] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.30.249.124 09:54:38.811 [DEBUG - New I/O worker #20] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.30.249.126 09:54:38.823 [DEBUG - New I/O worker #19] (Negotiator.java:764) Authenticated connection [id: 0x6475d4fc, /127.30.249.125:57810 => /127.30.249.125:55873] using SASL/PLAIN 09:54:38.831 [DEBUG - New I/O worker #20] (Negotiator.java:764) Authenticated connection [id: 0x88481761, /127.30.249.126:47711 => /127.30.249.126:55145] using SASL/PLAIN 09:54:38.831 [DEBUG - New I/O worker #21] (Negotiator.java:764) Authenticated connection [id: 0xd894d7e9, /127.30.249.124:60899 => /127.30.249.124:57750] using SASL/PLAIN 09:54:38.835 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:1986) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 09:54:38.835 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=29), Traces: [0ms] querying master, [1ms] Sub rpc: ConnectToMaster sending RPC to server master-127.30.249.125:55873, [1ms] Sub rpc: ConnectToMaster sending RPC to server master-127.30.249.126:55145, [2ms] Sub rpc: ConnectToMaster sending RPC to server master-127.30.249.124:57750, [20ms] Sub rpc: ConnectToMaster received from server master-127.30.249.125:55873 response OK, [29ms] Sub rpc: ConnectToMaster received from server master-127.30.249.126:55145 response OK, [29ms] Sub rpc: ConnectToMaster received from server master-127.30.249.124:57750 response OK, null) after lookup 09:54:38.836 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=retry RPC@1437596260 returned Deferred@279805621(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after retry RPC@1437596260), errback=(continuation of Deferred@1930531461 after retry RPC@1437596260)), so the following Deferred is getting paused: Deferred@1930531461(state=PAUSED, result=Deferred@279805621, callback=org.apache.kudu.client.AsyncKuduClient$1@21f7ac6d -> org.apache.kudu.client.AsyncKuduClient$2@ba11eed -> wakeup thread Test worker, errback=passthrough -> passthrough -> wakeup thread Test worker) 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.837302 31881 catalog_manager.cc:1376] Servicing CreateTable request from {username='jenkins'} at 127.30.249.124:60899: 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "org.apache.kudu.client.TestHybridTime-1542822859088" 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) schema { 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) columns { 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "key" 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) type: STRING 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) is_key: true 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) is_nullable: false 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) cfile_block_size: 0 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) partition_schema { 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) range_schema { 09:54:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) columns { 09:54:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) name: "key" 09:54:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) } 09:54:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:38.847985 31881 catalog_manager.cc:1527] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestHybridTime-1542822859088 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 09:54:38.867 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@21f7ac6d@569879661 returned Deferred@1887601146(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@ff8a0d5 -> (continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$1@21f7ac6d@569879661), errback=passthrough -> (continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$1@21f7ac6d@569879661)), so the following Deferred is getting paused: Deferred@1930531461(state=PAUSED, result=Deferred@1887601146, callback=org.apache.kudu.client.AsyncKuduClient$2@ba11eed -> wakeup thread Test worker, errback=passthrough -> wakeup thread Test worker) 09:54:38.868 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:737) Opened table 47e9981f4bcb4ac28906af457b74d2e6 09:54:38.868 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589)), so the following Deferred is getting paused: Deferred@1930531461(state=PAUSED, result=Deferred@326168963, callback=wakeup thread Test worker, errback=wakeup thread Test worker) 09:54:38.889 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030)), so the following Deferred is getting paused: Deferred@1959534025(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.896093 32006 tablet_service.cc:772] Processing CreateTablet for tablet 418b309de10f4796a3c73ecb59e5d00f (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=47e9981f4bcb4ac28906af457b74d2e6]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.896147 32137 tablet_service.cc:772] Processing CreateTablet for tablet 418b309de10f4796a3c73ecb59e5d00f (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=47e9981f4bcb4ac28906af457b74d2e6]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.897120 32006 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 418b309de10f4796a3c73ecb59e5d00f. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.897125 32137 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 418b309de10f4796a3c73ecb59e5d00f. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.903206 32269 tablet_service.cc:772] Processing CreateTablet for tablet 418b309de10f4796a3c73ecb59e5d00f (table=org.apache.kudu.client.TestHybridTime-1542822859088 [id=47e9981f4bcb4ac28906af457b74d2e6]), partition=RANGE (key) PARTITION UNBOUNDED 09:54:38.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.904219 32269 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 418b309de10f4796a3c73ecb59e5d00f. 1 dirs total, 0 dirs full, 0 dirs failed 09:54:38.909 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085)), so the following Deferred is getting paused: Deferred@714139448(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.918980 32006 ts_tablet_manager.cc:1173] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Registered tablet (data state: TABLET_DATA_READY) 09:54:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.922929 32137 ts_tablet_manager.cc:1173] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Registered tablet (data state: TABLET_DATA_READY) 09:54:38.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.924559 32269 ts_tablet_manager.cc:1173] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Registered tablet (data state: TABLET_DATA_READY) 09:54:38.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.925555 32373 ts_tablet_manager.cc:1028] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Bootstrapping tablet 09:54:38.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.927108 32373 tablet_bootstrap.cc:438] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Bootstrap starting. 09:54:38.929 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919)), so the following Deferred is getting paused: Deferred@2546273(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:38.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.931286 32375 ts_tablet_manager.cc:1028] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Bootstrapping tablet 09:54:38.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.931891 32375 tablet_bootstrap.cc:438] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Bootstrap starting. 09:54:38.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.932353 32373 tablet_bootstrap.cc:592] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: No blocks or log segments found. Creating new log. 09:54:38.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.934161 32373 log.cc:525] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Log is configured to *not* fsync() on all Append() calls 09:54:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.935534 32375 tablet_bootstrap.cc:592] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: No blocks or log segments found. Creating new log. 09:54:38.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.937749 32374 ts_tablet_manager.cc:1028] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Bootstrapping tablet 09:54:38.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.938503 32375 log.cc:525] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Log is configured to *not* fsync() on all Append() calls 09:54:38.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.938738 32374 tablet_bootstrap.cc:438] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Bootstrap starting. 09:54:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.942981 32374 tablet_bootstrap.cc:592] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: No blocks or log segments found. Creating new log. 09:54:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.944149 32373 tablet_bootstrap.cc:438] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: No bootstrap required, opened a new log 09:54:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.944635 32373 ts_tablet_manager.cc:1045] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Time spent bootstrapping tablet: real 0.018s user 0.008s sys 0.004s 09:54:38.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.944908 32374 log.cc:525] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Log is configured to *not* fsync() on all Append() calls 09:54:38.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.948238 32375 tablet_bootstrap.cc:438] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: No bootstrap required, opened a new log 09:54:38.948 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373)), so the following Deferred is getting paused: Deferred@1418658379(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.948966 32374 tablet_bootstrap.cc:438] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: No bootstrap required, opened a new log 09:54:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.949374 32374 ts_tablet_manager.cc:1045] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Time spent bootstrapping tablet: real 0.011s user 0.008s sys 0.000s 09:54:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.950126 32375 ts_tablet_manager.cc:1045] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.009s 09:54:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.958453 32373 raft_consensus.cc:339] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.959108 32373 raft_consensus.cc:365] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.959434 32373 raft_consensus.cc:704] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80ad009ef0e8437ba853f76a4c6113ec, State: Initialized, Role: FOLLOWER 09:54:38.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.960037 32373 consensus_queue.cc:229] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:38.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.964671 32373 ts_tablet_manager.cc:1073] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Time spent starting tablet: real 0.020s user 0.014s sys 0.002s 09:54:38.970 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826)), so the following Deferred is getting paused: Deferred@1932954623(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.969643 32375 raft_consensus.cc:339] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.970757 32375 raft_consensus.cc:365] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:38.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.971145 32375 raft_consensus.cc:704] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48d79176988849e0a1b5277b50c78b2a, State: Initialized, Role: FOLLOWER 09:54:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.972824 32375 consensus_queue.cc:229] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.976138 32334 heartbeater.cc:467] Master 127.30.249.124:57750 was elected leader, sending a full tablet report... 09:54:38.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.977795 32375 ts_tablet_manager.cc:1073] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Time spent starting tablet: real 0.027s user 0.023s sys 0.002s 09:54:38.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.981335 32374 raft_consensus.cc:339] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:38.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.982841 32374 raft_consensus.cc:365] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 09:54:38.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.983186 32374 raft_consensus.cc:704] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0717ab552dd7445b87e5ca48cb248966, State: Initialized, Role: FOLLOWER 09:54:38.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.984537 32374 consensus_queue.cc:229] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:38.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.986908 32208 heartbeater.cc:467] Master 127.30.249.124:57750 was elected leader, sending a full tablet report... 09:54:38.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:38.988920 32374 ts_tablet_manager.cc:1073] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Time spent starting tablet: real 0.039s user 0.022s sys 0.003s 09:54:38.990 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826) -> (continuation of Deferred@2050159494 after org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826) -> (continuation of Deferred@2050159494 after org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735)), so the following Deferred is getting paused: Deferred@2050159494(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:39.028 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@12881a96@310909590 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826) -> (continuation of Deferred@2050159494 after org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735) -> (continuation of Deferred@1436947019 after org.apache.kudu.client.AsyncKuduClient$13@12881a96@310909590), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826) -> (continuation of Deferred@2050159494 after org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735) -> (continuation of Deferred@1436947019 after org.apache.kudu.client.AsyncKuduClient$13@12881a96@310909590)), so the following Deferred is getting paused: Deferred@1436947019(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:39.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.059353 32077 tablet.cc:1766] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:39.069 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$13@1a987fbe@446201790 returned Deferred@326168963(state=PENDING, result=null, callback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826) -> (continuation of Deferred@2050159494 after org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735) -> (continuation of Deferred@1436947019 after org.apache.kudu.client.AsyncKuduClient$13@12881a96@310909590) -> (continuation of Deferred@2010841512 after org.apache.kudu.client.AsyncKuduClient$13@1a987fbe@446201790), errback=(continuation of Deferred@1930531461 after org.apache.kudu.client.AsyncKuduClient$2@ba11eed@195108589) -> (continuation of Deferred@1959534025 after org.apache.kudu.client.AsyncKuduClient$13@10a5d376@279303030) -> (continuation of Deferred@714139448 after org.apache.kudu.client.AsyncKuduClient$13@3ef8e58d@1056499085) -> (continuation of Deferred@2546273 after org.apache.kudu.client.AsyncKuduClient$13@1410bf7f@336641919) -> (continuation of Deferred@1418658379 after org.apache.kudu.client.AsyncKuduClient$13@798b9365@2039190373) -> (continuation of Deferred@1932954623 after org.apache.kudu.client.AsyncKuduClient$13@4575a4e2@1165337826) -> (continuation of Deferred@2050159494 after org.apache.kudu.client.AsyncKuduClient$13@56833e47@1451441735) -> (continuation of Deferred@1436947019 after org.apache.kudu.client.AsyncKuduClient$13@12881a96@310909590) -> (continuation of Deferred@2010841512 after org.apache.kudu.client.AsyncKuduClient$13@1a987fbe@446201790)), so the following Deferred is getting paused: Deferred@2010841512(state=PAUSED, result=Deferred@326168963, callback=, errback=) 09:54:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.102932 32211 tablet.cc:1766] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.123984 32379 raft_consensus.cc:471] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 09:54:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.124250 32379 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Snoozing failure detection for 2.202s (starting election) 09:54:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.124529 32379 raft_consensus.cc:493] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.125562 32379 leader_election.cc:251] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.126088 32379 leader_election.cc:251] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 pre-election: Requesting pre-vote from peer 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.137300 32289 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "418b309de10f4796a3c73ecb59e5d00f" candidate_uuid: "80ad009ef0e8437ba853f76a4c6113ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d79176988849e0a1b5277b50c78b2a" is_pre_election: true 09:54:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.137841 32289 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Snoozing failure detection for 1.551s (vote granted) 09:54:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.138083 32289 raft_consensus.cc:2352] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 80ad009ef0e8437ba853f76a4c6113ec in term 0. 09:54:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.139379 31946 leader_election.cc:387] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 pre-election: Vote granted by peer 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.139778 31946 leader_election.cc:278] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. 09:54:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.139983 32157 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "418b309de10f4796a3c73ecb59e5d00f" candidate_uuid: "80ad009ef0e8437ba853f76a4c6113ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0717ab552dd7445b87e5ca48cb248966" is_pre_election: true 09:54:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.140753 32379 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Snoozing failure detection for 1.625s (election complete) 09:54:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.140684 32157 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Snoozing failure detection for 2.171s (vote granted) 09:54:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.140961 32379 raft_consensus.cc:2646] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Leader pre-election won for term 1 09:54:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.141052 32157 raft_consensus.cc:2352] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 80ad009ef0e8437ba853f76a4c6113ec in term 0. 09:54:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.141204 32379 raft_consensus.cc:471] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 09:54:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.141412 32379 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Snoozing failure detection for 1.511s (starting election) 09:54:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.141618 32379 raft_consensus.cc:2892] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 0 FOLLOWER]: Advancing to term 1 09:54:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.142015 31946 leader_election.cc:387] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 pre-election: Vote granted by peer 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.145092 32379 raft_consensus.cc:493] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.145874 32379 leader_election.cc:251] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 election: Requesting vote from peer 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.146342 32379 leader_election.cc:251] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 election: Requesting vote from peer 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.147403 32289 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "418b309de10f4796a3c73ecb59e5d00f" candidate_uuid: "80ad009ef0e8437ba853f76a4c6113ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d79176988849e0a1b5277b50c78b2a" 09:54:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.147867 32289 raft_consensus.cc:2892] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 0 FOLLOWER]: Advancing to term 1 09:54:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.147989 32157 tablet_service.cc:1008] Received RequestConsensusVote() RPC: tablet_id: "418b309de10f4796a3c73ecb59e5d00f" candidate_uuid: "80ad009ef0e8437ba853f76a4c6113ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0717ab552dd7445b87e5ca48cb248966" 09:54:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.148156 32289 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Snoozing failure detection for 1.955s (vote granted) 09:54:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.148417 32157 raft_consensus.cc:2892] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 0 FOLLOWER]: Advancing to term 1 09:54:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.148702 32157 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966: Snoozing failure detection for 2.061s (vote granted) 09:54:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.153021 32289 raft_consensus.cc:2352] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 80ad009ef0e8437ba853f76a4c6113ec in term 1. 09:54:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.153237 32157 raft_consensus.cc:2352] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 80ad009ef0e8437ba853f76a4c6113ec in term 1. 09:54:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.154023 31946 leader_election.cc:387] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 election: Vote granted by peer 48d79176988849e0a1b5277b50c78b2a (127.30.249.67:53351) 09:54:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.154217 31946 leader_election.cc:278] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. 09:54:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.154600 32379 raft_consensus.cc:2840] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec: Snoozing failure detection for 1.660s (election complete) 09:54:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.154824 31946 leader_election.cc:387] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [CANDIDATE]: Term 1 election: Vote granted by peer 0717ab552dd7445b87e5ca48cb248966 (127.30.249.66:54184) 09:54:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.154814 32379 raft_consensus.cc:2646] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 1 FOLLOWER]: Leader election won for term 1 09:54:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.156301 32379 raft_consensus.cc:666] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [term 1 LEADER]: Becoming Leader. State: Replica: 80ad009ef0e8437ba853f76a4c6113ec, State: Running, Role: LEADER 09:54:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.156981 32379 consensus_queue.cc:206] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } } 09:54:39.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.166795 31882 catalog_manager.cc:3781] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec reported cstate change: term changed from 0 to 1, leader changed from to 80ad009ef0e8437ba853f76a4c6113ec (127.30.249.65). New cstate: current_term: 1 leader_uuid: "80ad009ef0e8437ba853f76a4c6113ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "80ad009ef0e8437ba853f76a4c6113ec" member_type: VOTER last_known_addr { host: "127.30.249.65" port: 59307 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 } health_report { overall_health: UNKNOWN } } } 09:54:39.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.207420 32339 tablet.cc:1766] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a: Can't schedule compaction. Clean time has not been advanced past its initial value. 09:54:39.232 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:1986) Learned about tablet 418b309de10f4796a3c73ecb59e5d00f for table 'org.apache.kudu.client.TestHybridTime-1542822859088' with partition [, ) 09:54:39.232 [DEBUG - New I/O worker #21] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=418b309de10f4796a3c73ecb59e5d00f@[0717ab552dd7445b87e5ca48cb248966(127.30.249.66:54184),48d79176988849e0a1b5277b50c78b2a(127.30.249.67:53351),80ad009ef0e8437ba853f76a4c6113ec(127.30.249.65:59307)[L]]}] 09:54:39.232 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:1150) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=2), Traces: [0ms] querying master, [0ms] Sub rpc: GetTableLocations sending RPC to server master-127.30.249.124:57750, [2ms] Sub rpc: GetTableLocations received from server master-127.30.249.124:57750 response OK, null) after lookup 09:54:39.233 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=retry RPC@1082430404 returned Deferred@431518181(state=PENDING, result=null, callback=(continuation of Deferred@918676795 after retry RPC@1082430404), errback=(continuation of Deferred@918676795 after retry RPC@1082430404)), so the following Deferred is getting paused: Deferred@918676795(state=PAUSED, result=Deferred@431518181, callback=org.apache.kudu.client.AsyncKuduSession$2@3c55ec0d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@582b52f0 -> wakeup thread Time-limited test) 09:54:39.247 [DEBUG - New I/O worker #22] (Negotiator.java:764) Authenticated connection [id: 0x438d2ee1, /127.30.249.65:38603 => /127.30.249.65:59307] using TOKEN/null 09:54:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.255720 32379 consensus_queue.cc:745] Safe time advancement without writes is disabled. Snapshot reads on non-leader replicas may stall if there are no writes in progress. 09:54:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.257897 32157 raft_consensus.cc:1180] T 418b309de10f4796a3c73ecb59e5d00f P 0717ab552dd7445b87e5ca48cb248966 [term 1 FOLLOWER]: Refusing update from remote peer 80ad009ef0e8437ba853f76a4c6113ec: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.257998 32289 raft_consensus.cc:1180] T 418b309de10f4796a3c73ecb59e5d00f P 48d79176988849e0a1b5277b50c78b2a [term 1 FOLLOWER]: Refusing update from remote peer 80ad009ef0e8437ba853f76a4c6113ec: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 09:54:39.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.259666 32384 consensus_queue.cc:976] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "48d79176988849e0a1b5277b50c78b2a" member_type: VOTER last_known_addr { host: "127.30.249.67" port: 53351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:39.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.260231 32379 consensus_queue.cc:976] T 418b309de10f4796a3c73ecb59e5d00f P 80ad009ef0e8437ba853f76a4c6113ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "0717ab552dd7445b87e5ca48cb248966" member_type: VOTER last_known_addr { host: "127.30.249.66" port: 54184 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 09:54:39.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.280419 32387 mvcc.cc:203] Tried to move safe_time back from 6319402513427058688 to 6319402513033138176. Current Snapshot: MvccSnapshot[committed={T|T < 6319402513427058688}] 09:54:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.289407 32388 mvcc.cc:203] Tried to move safe_time back from 6319402513427058688 to 6319402513033138176. Current Snapshot: MvccSnapshot[committed={T|T < 6319402513427058688}] 09:54:39.291 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[0]: Wed Nov 21 09:54:39 PST 2018 Logical value: 0 09:54:39.308 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[1]: Wed Nov 21 09:54:44 PST 2018 Logical value: 1 09:54:39.317 [DEBUG - Time-limited test] (TestHybridTime.java:116) Clock value after write[2]: Wed Nov 21 09:54:44 PST 2018 Logical value: 4 09:54:39.318 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@1663807440(state=PENDING, result=null, callback=, errback=)} 09:54:39.318 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@6714276d@1729374061 returned Deferred@571909611(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@756386064 after org.apache.kudu.client.AsyncKuduSession$1@6714276d@1729374061), errback=passthrough -> (continuation of Deferred@756386064 after org.apache.kudu.client.AsyncKuduSession$1@6714276d@1729374061)), so the following Deferred is getting paused: Deferred@756386064(state=PAUSED, result=Deferred@571909611, callback=, errback=) 09:54:39.326 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[0]: Wed Nov 21 09:54:44 PST 2018 Logical value: 5 09:54:39.327 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@327083597(state=PENDING, result=null, callback=, errback=)} 09:54:39.327 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@55979400@1435997184 returned Deferred@1111055012(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@1663807440 after org.apache.kudu.client.AsyncKuduSession$1@55979400@1435997184), errback=passthrough -> (continuation of Deferred@1663807440 after org.apache.kudu.client.AsyncKuduSession$1@55979400@1435997184)), so the following Deferred is getting paused: Deferred@1663807440(state=PAUSED, result=Deferred@1111055012, callback=, errback=) 09:54:39.337 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[1]: Wed Nov 21 09:54:44 PST 2018 Logical value: 6 09:54:39.338 [DEBUG - Time-limited test] (AsyncKuduSession.java:452) flushing buffer: Buffer{operations=1, flusherTask=null, flushNotification=Deferred@1130698490(state=PENDING, result=null, callback=, errback=)} 09:54:39.338 [DEBUG - Time-limited test] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduSession$1@274dcc55@659410005 returned Deferred@1318288168(state=PENDING, result=null, callback=ConvertBatchToListOfResponsesCB -> (continuation of Deferred@327083597 after org.apache.kudu.client.AsyncKuduSession$1@274dcc55@659410005), errback=passthrough -> (continuation of Deferred@327083597 after org.apache.kudu.client.AsyncKuduSession$1@274dcc55@659410005)), so the following Deferred is getting paused: Deferred@327083597(state=PAUSED, result=Deferred@1318288168, callback=, errback=) 09:54:39.349 [DEBUG - Time-limited test] (TestHybridTime.java:152) Clock value after write[2]: Wed Nov 21 09:54:44 PST 2018 Logical value: 7 09:54:39.357 [DEBUG - New I/O worker #22] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=418b309de10f4796a3c73ecb59e5d00f, scannerId=null, scanRequestTimeout=30000) 09:54:39.362 [DEBUG - New I/O worker #22] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 1 rows, more = false, responseScanTimestamp = 6319402533907058688) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=418b309de10f4796a3c73ecb59e5d00f, scannerId=null, scanRequestTimeout=30000) 09:54:39.367 [DEBUG - New I/O worker #22] (AsyncKuduScanner.java:1037) AsyncKuduScanner$Response(scannerId = "", data = RowResultIterator for 3 rows, more = false, responseScanTimestamp = 6319402533907058693) for scanner KuduScanner(table=org.apache.kudu.client.TestHybridTime-1542822859088, tablet=418b309de10f4796a3c73ecb59e5d00f, scannerId=null, scanRequestTimeout=30000) 09:54:39.368 [DEBUG - New I/O worker #19] (Connection.java:688) [peer master-127.30.249.125:55873(127.30.249.125:55873)] cleaning up while in state READY due to: connection disconnected 09:54:39.369 [DEBUG - New I/O worker #20] (Connection.java:688) [peer master-127.30.249.126:55145(127.30.249.126:55145)] cleaning up while in state READY due to: connection disconnected 09:54:39.371 [DEBUG - New I/O worker #21] (Connection.java:688) [peer master-127.30.249.124:57750(127.30.249.124:57750)] cleaning up while in state READY due to: connection disconnected 09:54:39.372 [DEBUG - Test worker] (AsyncKuduClient.java:2132) Releasing all remaining resources 09:54:39.372 [DEBUG - New I/O worker #22] (Connection.java:688) [peer 80ad009ef0e8437ba853f76a4c6113ec(127.30.249.65:59307)] cleaning up while in state READY due to: connection disconnected 09:54:39.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.378692 31717 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 31916 09:54:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.403161 31734 connection.cc:512] server connection from 127.30.249.65:60307 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.65:60307): Connection reset by peer (error 104) 09:54:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.404043 31863 connection.cc:512] server connection from 127.30.249.65:38364 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.65:38364): Connection reset by peer (error 104) 09:54:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.404091 31801 connection.cc:512] server connection from 127.30.249.65:39207 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.65:39207): Connection reset by peer (error 104) 09:54:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.404542 32221 connection.cc:512] server connection from 127.30.249.65:32834 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.65:32834): Connection reset by peer (error 104) 09:54:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.405230 32087 connection.cc:512] server connection from 127.30.249.65:40076 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.65:40076): Connection reset by peer (error 104) 09:54:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.406209 31717 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 32073 09:54:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.424311 31733 connection.cc:512] server connection from 127.30.249.66:50084 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.66:50084): Connection reset by peer (error 104) 09:54:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.424305 31863 connection.cc:512] server connection from 127.30.249.66:52533 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.66:52533): Connection reset by peer (error 104) 09:54:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.425062 31717 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 32205 09:54:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.425400 31799 connection.cc:512] server connection from 127.30.249.66:58875 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.66:58875): Connection reset by peer (error 104) 09:54:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.444954 31801 connection.cc:512] server connection from 127.30.249.67:57015 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.67:57015): Connection reset by peer (error 104) 09:54:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.445386 31735 connection.cc:512] server connection from 127.30.249.67:60568 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.67:60568): Connection reset by peer (error 104) 09:54:39.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) W1121 09:54:39.445952 31863 connection.cc:512] server connection from 127.30.249.67:33354 recv error: Network error: failed to read from TLS socket (remote: 127.30.249.67:33354): Connection reset by peer (error 104) 09:54:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.446852 31717 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 31725 09:54:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.467679 31717 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 31787 09:54:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:535) I1121 09:54:39.487782 31717 external_mini_cluster.cc:1043] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 31851 09:54:40.526 [ERROR - Test worker] (RetryRule.java:76) test(org.apache.kudu.client.TestHybridTime): failed run 3 java.lang.AssertionError: expected:<4> but was:<3> at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.failNotEquals(Assert.java:834) at org.junit.Assert.assertEquals(Assert.java:645) at org.junit.Assert.assertEquals(Assert.java:631) at org.apache.kudu.client.TestHybridTime.test(TestHybridTime.java:167) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 09:54:40.527 [ERROR - Test worker] (RetryRule.java:79) test(org.apache.kudu.client.TestHybridTime): giving up after 3 failures ]]>