/127.1.193.124:58247] using SASL/PLAIN 01:26:42.188 [DEBUG - New I/O worker #1] (Negotiator.java:764) Authenticated connection [id: 0xf9f52f77, /127.1.193.125:50063 => /127.1.193.125:37435] using SASL/PLAIN 01:26:42.189 [DEBUG - New I/O worker #3] (Negotiator.java:764) Authenticated connection [id: 0xf1d8c215, /127.1.193.126:50802 => /127.1.193.126:40739] using SASL/PLAIN 01:26:42.230 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:42.232 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=474), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [69ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [70ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [438ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:42.246 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=retry RPC@91201180 returned Deferred@2085307926(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after retry RPC@91201180), errback=(continuation of Deferred@1299770537 after retry RPC@91201180)), so the following Deferred is getting paused: Deferred@1299770537(state=PAUSED, result=Deferred@2085307926, callback=org.apache.kudu.client.AsyncKuduClient$1@185893a4 -> org.apache.kudu.client.AsyncKuduClient$2@61ba754d -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 01:26:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.246130 1901 catalog_manager.cc:1386] Servicing CreateTable request from {username='jenkins'} at 127.1.193.125:50063: 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "testManyClients" 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) schema { 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: true 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column1_i" 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column2_i" 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:26:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column3_s" 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: STRING 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: true 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) encoding: DICT_ENCODING 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) compression: LZ4 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 4096 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column4_b" 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: BOOL 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) split_rows_range_bounds { 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) rows: "\004\001\000\n\000\000\000\004\001\000\024\000\000\000\004\001\000\036\000\000\000" 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) partition_schema { 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) range_schema { 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:26:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.266496 1901 catalog_manager.cc:1537] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table testManyClients in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 01:26:42.305 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@185893a4@408458148 returned Deferred@1206383770(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@4df0456b -> (continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$1@185893a4@408458148), errback=passthrough -> (continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$1@185893a4@408458148)), so the following Deferred is getting paused: Deferred@1299770537(state=PAUSED, result=Deferred@1206383770, callback=org.apache.kudu.client.AsyncKuduClient$2@61ba754d -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:26:42.317 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:42.318 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677)), so the following Deferred is getting paused: Deferred@1299770537(state=PAUSED, result=Deferred@1534084728, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 01:26:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.333855 2352 tablet_service.cc:870] Processing CreateTablet for tablet 02d12d6ae86c466fb511f0ddef2f6fc7 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION VALUES < 10 01:26:42.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.337776 2220 tablet_service.cc:870] Processing CreateTablet for tablet 02d12d6ae86c466fb511f0ddef2f6fc7 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION VALUES < 10 01:26:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.339648 2219 tablet_service.cc:870] Processing CreateTablet for tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION 10 <= VALUES < 20 01:26:42.344 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600)), so the following Deferred is getting paused: Deferred@1283689476(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.341307 2218 tablet_service.cc:870] Processing CreateTablet for tablet c0626d9f34c84a0488641522977ae3d0 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION 20 <= VALUES < 30 01:26:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.342952 2217 tablet_service.cc:870] Processing CreateTablet for tablet 8062ddd6695848d6b53496b0c2a33e88 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION VALUES >= 30 01:26:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.344456 2220 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 02d12d6ae86c466fb511f0ddef2f6fc7. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.345157 2219 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.346433 2352 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 02d12d6ae86c466fb511f0ddef2f6fc7. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.348111 2217 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 8062ddd6695848d6b53496b0c2a33e88. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.349203 2218 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet c0626d9f34c84a0488641522977ae3d0. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.351084 2088 tablet_service.cc:870] Processing CreateTablet for tablet 02d12d6ae86c466fb511f0ddef2f6fc7 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION VALUES < 10 01:26:42.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.352592 2088 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 02d12d6ae86c466fb511f0ddef2f6fc7. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.373 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630)), so the following Deferred is getting paused: Deferred@1862630731(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.387871 2219 ts_tablet_manager.cc:1175] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.394 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824)), so the following Deferred is getting paused: Deferred@255430681(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.391602 2218 ts_tablet_manager.cc:1175] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.397253 2217 ts_tablet_manager.cc:1175] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.404448 2220 ts_tablet_manager.cc:1175] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.406076 2460 ts_tablet_manager.cc:1030] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Bootstrapping tablet 01:26:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.406955 2460 tablet_bootstrap.cc:438] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Bootstrap starting. 01:26:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.414355 2088 ts_tablet_manager.cc:1175] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.420 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838)), so the following Deferred is getting paused: Deferred@11903467(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.423490 2352 ts_tablet_manager.cc:1175] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.425248 2460 tablet_bootstrap.cc:592] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: No blocks or log segments found. Creating new log. 01:26:42.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.428617 2460 log.cc:525] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Log is configured to *not* fsync() on all Append() calls 01:26:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.336468 2349 tablet_service.cc:870] Processing CreateTablet for tablet 8062ddd6695848d6b53496b0c2a33e88 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION VALUES >= 30 01:26:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.439239 2349 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 8062ddd6695848d6b53496b0c2a33e88. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.439827 2460 tablet_bootstrap.cc:438] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: No bootstrap required, opened a new log 01:26:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.440326 2460 ts_tablet_manager.cc:1047] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Time spent bootstrapping tablet: real 0.034s user 0.011s sys 0.000s 01:26:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.451314 2349 ts_tablet_manager.cc:1175] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.455 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185)), so the following Deferred is getting paused: Deferred@1257159962(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.344245 2351 tablet_service.cc:870] Processing CreateTablet for tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION 10 <= VALUES < 20 01:26:42.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.345839 2350 tablet_service.cc:870] Processing CreateTablet for tablet c0626d9f34c84a0488641522977ae3d0 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION 20 <= VALUES < 30 01:26:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.466913 2351 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.468962 2460 raft_consensus.cc:340] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.470050 2460 raft_consensus.cc:366] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.470224 2350 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet c0626d9f34c84a0488641522977ae3d0. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.470295 2460 raft_consensus.cc:705] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 134794f36230463ea5d0f3f800c08596, State: Initialized, Role: FOLLOWER 01:26:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.471216 2460 consensus_queue.cc:229] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.487383 2351 ts_tablet_manager.cc:1175] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.493387 2460 ts_tablet_manager.cc:1075] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Time spent starting tablet: real 0.053s user 0.023s sys 0.007s 01:26:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.495272 2460 ts_tablet_manager.cc:1030] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: Bootstrapping tablet 01:26:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.496006 2460 tablet_bootstrap.cc:438] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: Bootstrap starting. 01:26:42.498 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506)), so the following Deferred is getting paused: Deferred@1355300155(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.495703 2350 ts_tablet_manager.cc:1175] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.507800 2464 raft_consensus.cc:472] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:42.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.508275 2464 raft_consensus.cc:494] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.509377 2460 tablet_bootstrap.cc:592] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: No blocks or log segments found. Creating new log. 01:26:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.511539 2464 leader_election.cc:296] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533), b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179) 01:26:42.535 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478)), so the following Deferred is getting paused: Deferred@1151674698(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.541020 2174 leader_election.cc:348] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533): Not found: Tablet not found: 71e4dc0a850f4442b6c11bfcaa1b5ce8 01:26:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.537330 2460 tablet_bootstrap.cc:438] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: No bootstrap required, opened a new log 01:26:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.544829 2460 ts_tablet_manager.cc:1047] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: Time spent bootstrapping tablet: real 0.049s user 0.005s sys 0.013s 01:26:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.553447 2460 raft_consensus.cc:340] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.554771 2460 raft_consensus.cc:366] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.555533 2460 raft_consensus.cc:705] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 134794f36230463ea5d0f3f800c08596, State: Initialized, Role: FOLLOWER 01:26:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.556965 2460 consensus_queue.cc:229] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.570006 2292 tablet.cc:1783] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:26:42.573 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478) -> (continuation of Deferred@1396756269 after org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478) -> (continuation of Deferred@1396756269 after org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245)), so the following Deferred is getting paused: Deferred@1396756269(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.575780 2460 ts_tablet_manager.cc:1075] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596: Time spent starting tablet: real 0.024s user 0.005s sys 0.000s 01:26:42.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.576539 2460 ts_tablet_manager.cc:1030] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: Bootstrapping tablet 01:26:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.577317 2460 tablet_bootstrap.cc:438] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: Bootstrap starting. 01:26:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.553453 2372 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "71e4dc0a850f4442b6c11bfcaa1b5ce8" candidate_uuid: "134794f36230463ea5d0f3f800c08596" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" is_pre_election: true 01:26:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.581383 2460 tablet_bootstrap.cc:592] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: No blocks or log segments found. Creating new log. 01:26:42.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.583585 2170 leader_election.cc:348] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179): Illegal state: must be running to vote when last-logged opid is not known 01:26:42.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.584076 2170 leader_election.cc:310] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 134794f36230463ea5d0f3f800c08596; no voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b 01:26:42.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.584959 2464 raft_consensus.cc:2588] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority 01:26:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.593940 2460 tablet_bootstrap.cc:438] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: No bootstrap required, opened a new log 01:26:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.594630 2460 ts_tablet_manager.cc:1047] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.006s 01:26:42.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.596931 2460 raft_consensus.cc:340] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.597983 2460 raft_consensus.cc:366] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.598528 2460 raft_consensus.cc:705] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 134794f36230463ea5d0f3f800c08596, State: Initialized, Role: FOLLOWER 01:26:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.599530 2460 consensus_queue.cc:229] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.613801 2460 ts_tablet_manager.cc:1075] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596: Time spent starting tablet: real 0.019s user 0.002s sys 0.004s 01:26:42.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.614765 2460 ts_tablet_manager.cc:1030] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: Bootstrapping tablet 01:26:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.615569 2460 tablet_bootstrap.cc:438] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: Bootstrap starting. 01:26:42.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.619514 2460 tablet_bootstrap.cc:592] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: No blocks or log segments found. Creating new log. 01:26:42.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.629006 2460 tablet_bootstrap.cc:438] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: No bootstrap required, opened a new log 01:26:42.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.629809 2460 ts_tablet_manager.cc:1047] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: Time spent bootstrapping tablet: real 0.014s user 0.007s sys 0.005s 01:26:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.633138 2464 raft_consensus.cc:472] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.632314 2460 raft_consensus.cc:340] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.633568 2464 raft_consensus.cc:494] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.635602 2464 leader_election.cc:296] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533), b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179) 01:26:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.637202 2460 raft_consensus.cc:366] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.637495 2372 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8062ddd6695848d6b53496b0c2a33e88" candidate_uuid: "134794f36230463ea5d0f3f800c08596" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" is_pre_election: true 01:26:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.637774 2460 raft_consensus.cc:705] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 134794f36230463ea5d0f3f800c08596, State: Initialized, Role: FOLLOWER 01:26:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.639950 2170 leader_election.cc:348] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179): Illegal state: must be running to vote when last-logged opid is not known 01:26:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.640333 2174 leader_election.cc:348] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533): Not found: Tablet not found: 8062ddd6695848d6b53496b0c2a33e88 01:26:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.640652 2174 leader_election.cc:310] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 134794f36230463ea5d0f3f800c08596; no voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b 01:26:42.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.641994 2464 raft_consensus.cc:2588] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority 01:26:42.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.642765 2460 consensus_queue.cc:229] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.652216 2460 ts_tablet_manager.cc:1075] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596: Time spent starting tablet: real 0.022s user 0.005s sys 0.001s 01:26:42.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.354130 2087 tablet_service.cc:870] Processing CreateTablet for tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION 10 <= VALUES < 20 01:26:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.681505 2087 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.360052 2085 tablet_service.cc:870] Processing CreateTablet for tablet 8062ddd6695848d6b53496b0c2a33e88 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION VALUES >= 30 01:26:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.695703 2085 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 8062ddd6695848d6b53496b0c2a33e88. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.360672 2086 tablet_service.cc:870] Processing CreateTablet for tablet c0626d9f34c84a0488641522977ae3d0 (table=testManyClients [id=9829e8c5872f49779e6db7a28aaa0015]), partition=RANGE (key) PARTITION 20 <= VALUES < 30 01:26:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.707671 2086 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet c0626d9f34c84a0488641522977ae3d0. 1 dirs total, 0 dirs full, 0 dirs failed 01:26:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.714229 2461 ts_tablet_manager.cc:1030] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrapping tablet 01:26:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.420025 2415 heartbeater.cc:467] Master 127.1.193.125:37435 was elected leader, sending a full tablet report... 01:26:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.719682 2462 ts_tablet_manager.cc:1030] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrapping tablet 01:26:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.720468 2462 tablet_bootstrap.cc:438] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrap starting. 01:26:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.721951 2461 tablet_bootstrap.cc:438] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrap starting. 01:26:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.721069 2087 ts_tablet_manager.cc:1175] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.727092 2462 tablet_bootstrap.cc:592] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: No blocks or log segments found. Creating new log. 01:26:42.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.729962 2462 log.cc:525] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Log is configured to *not* fsync() on all Append() calls 01:26:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.731421 2461 tablet_bootstrap.cc:592] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: No blocks or log segments found. Creating new log. 01:26:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.732282 2085 ts_tablet_manager.cc:1175] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.736546 2464 raft_consensus.cc:472] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.737085 2464 raft_consensus.cc:494] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.737773 2086 ts_tablet_manager.cc:1175] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: Registered tablet (data state: TABLET_DATA_READY) 01:26:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.739208 2464 leader_election.cc:296] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179), 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533) 01:26:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.735854 2461 log.cc:525] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Log is configured to *not* fsync() on all Append() calls 01:26:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.743531 2462 tablet_bootstrap.cc:438] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: No bootstrap required, opened a new log 01:26:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.744431 2462 ts_tablet_manager.cc:1047] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent bootstrapping tablet: real 0.024s user 0.009s sys 0.007s 01:26:42.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.745123 2372 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "02d12d6ae86c466fb511f0ddef2f6fc7" candidate_uuid: "134794f36230463ea5d0f3f800c08596" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" is_pre_election: true 01:26:42.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.748651 2461 tablet_bootstrap.cc:438] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: No bootstrap required, opened a new log 01:26:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.749421 2461 ts_tablet_manager.cc:1047] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent bootstrapping tablet: real 0.028s user 0.009s sys 0.007s 01:26:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.751631 2170 leader_election.cc:348] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179): Illegal state: must be running to vote when last-logged opid is not known 01:26:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.741459 2108 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "02d12d6ae86c466fb511f0ddef2f6fc7" candidate_uuid: "134794f36230463ea5d0f3f800c08596" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" is_pre_election: true 01:26:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.758647 2174 leader_election.cc:348] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533): Illegal state: must be running to vote when last-logged opid is not known 01:26:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.759024 2174 leader_election.cc:310] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 134794f36230463ea5d0f3f800c08596; no voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b 01:26:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.759835 2464 raft_consensus.cc:2588] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority 01:26:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.760689 2462 raft_consensus.cc:340] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.761935 2462 raft_consensus.cc:366] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.762213 2462 raft_consensus.cc:705] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Initialized, Role: FOLLOWER 01:26:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.763686 2462 consensus_queue.cc:229] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.765381 2461 raft_consensus.cc:340] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.766777 2461 raft_consensus.cc:366] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.767210 2461 raft_consensus.cc:705] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6eaa0da28e264c548dcf48b8ea8f690e, State: Initialized, Role: FOLLOWER 01:26:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.768024 2461 consensus_queue.cc:229] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.773149 2462 ts_tablet_manager.cc:1075] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent starting tablet: real 0.028s user 0.017s sys 0.006s 01:26:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.773711 2462 ts_tablet_manager.cc:1030] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrapping tablet 01:26:42.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.774353 2462 tablet_bootstrap.cc:438] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrap starting. 01:26:42.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.780434 2462 tablet_bootstrap.cc:592] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: No blocks or log segments found. Creating new log. 01:26:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.781852 2461 ts_tablet_manager.cc:1075] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent starting tablet: real 0.032s user 0.013s sys 0.008s 01:26:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.782420 2461 ts_tablet_manager.cc:1030] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrapping tablet 01:26:42.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.789908 2461 tablet_bootstrap.cc:438] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrap starting. 01:26:42.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.795189 2462 tablet_bootstrap.cc:438] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: No bootstrap required, opened a new log 01:26:42.795 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@4aaf9eb6@1253023414 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478) -> (continuation of Deferred@1396756269 after org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245) -> (continuation of Deferred@657261008 after org.apache.kudu.client.AsyncKuduClient$15@4aaf9eb6@1253023414), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478) -> (continuation of Deferred@1396756269 after org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245) -> (continuation of Deferred@657261008 after org.apache.kudu.client.AsyncKuduClient$15@4aaf9eb6@1253023414)), so the following Deferred is getting paused: Deferred@657261008(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.796139 2462 ts_tablet_manager.cc:1047] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent bootstrapping tablet: real 0.022s user 0.005s sys 0.007s 01:26:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.798074 2461 tablet_bootstrap.cc:592] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: No blocks or log segments found. Creating new log. 01:26:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.798506 2462 raft_consensus.cc:340] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.799290 2462 raft_consensus.cc:366] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.799537 2462 raft_consensus.cc:705] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Initialized, Role: FOLLOWER 01:26:42.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.800324 2462 consensus_queue.cc:229] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.802881 2462 ts_tablet_manager.cc:1075] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s 01:26:42.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.803457 2462 ts_tablet_manager.cc:1030] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrapping tablet 01:26:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.804102 2462 tablet_bootstrap.cc:438] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrap starting. 01:26:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.805949 2461 tablet_bootstrap.cc:438] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: No bootstrap required, opened a new log 01:26:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.806622 2461 ts_tablet_manager.cc:1047] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent bootstrapping tablet: real 0.017s user 0.006s sys 0.001s 01:26:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.807698 2462 tablet_bootstrap.cc:592] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: No blocks or log segments found. Creating new log. 01:26:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.809033 2461 raft_consensus.cc:340] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.810566 2461 raft_consensus.cc:366] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.811091 2461 raft_consensus.cc:705] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6eaa0da28e264c548dcf48b8ea8f690e, State: Initialized, Role: FOLLOWER 01:26:42.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.812108 2461 consensus_queue.cc:229] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.818158 2461 ts_tablet_manager.cc:1075] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent starting tablet: real 0.011s user 0.005s sys 0.000s 01:26:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.820169 2462 tablet_bootstrap.cc:438] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: No bootstrap required, opened a new log 01:26:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.822804 2461 ts_tablet_manager.cc:1030] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrapping tablet 01:26:42.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.824482 2462 ts_tablet_manager.cc:1047] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent bootstrapping tablet: real 0.021s user 0.006s sys 0.006s 01:26:42.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.824314 2461 tablet_bootstrap.cc:438] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrap starting. 01:26:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.826720 2462 raft_consensus.cc:340] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.827651 2462 raft_consensus.cc:366] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.828066 2462 raft_consensus.cc:705] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Initialized, Role: FOLLOWER 01:26:42.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.830133 2462 consensus_queue.cc:229] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.832654 2462 ts_tablet_manager.cc:1075] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent starting tablet: real 0.008s user 0.003s sys 0.001s 01:26:42.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.833261 2461 tablet_bootstrap.cc:592] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: No blocks or log segments found. Creating new log. 01:26:42.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.833257 2462 ts_tablet_manager.cc:1030] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrapping tablet 01:26:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.833982 2462 tablet_bootstrap.cc:438] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: Bootstrap starting. 01:26:42.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.838905 2462 tablet_bootstrap.cc:592] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: No blocks or log segments found. Creating new log. 01:26:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.842089 2422 tablet.cc:1783] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:26:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.847867 2461 tablet_bootstrap.cc:438] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: No bootstrap required, opened a new log 01:26:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.848356 2461 ts_tablet_manager.cc:1047] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent bootstrapping tablet: real 0.024s user 0.009s sys 0.000s 01:26:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.850179 2461 raft_consensus.cc:340] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.851114 2461 raft_consensus.cc:366] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.852198 2461 raft_consensus.cc:705] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6eaa0da28e264c548dcf48b8ea8f690e, State: Initialized, Role: FOLLOWER 01:26:42.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.852948 2461 consensus_queue.cc:229] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.854184 2462 tablet_bootstrap.cc:438] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: No bootstrap required, opened a new log 01:26:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.854806 2462 ts_tablet_manager.cc:1047] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent bootstrapping tablet: real 0.021s user 0.009s sys 0.003s 01:26:42.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.858582 2461 ts_tablet_manager.cc:1075] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent starting tablet: real 0.010s user 0.004s sys 0.001s 01:26:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.857290 2462 raft_consensus.cc:340] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.860393 2462 raft_consensus.cc:366] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.860787 2462 raft_consensus.cc:705] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Initialized, Role: FOLLOWER 01:26:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.862504 2461 ts_tablet_manager.cc:1030] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrapping tablet 01:26:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.861678 2462 consensus_queue.cc:229] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.863919 2461 tablet_bootstrap.cc:438] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: Bootstrap starting. 01:26:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.867468 2462 ts_tablet_manager.cc:1075] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b: Time spent starting tablet: real 0.012s user 0.005s sys 0.000s 01:26:42.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.870278 2461 tablet_bootstrap.cc:592] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: No blocks or log segments found. Creating new log. 01:26:42.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.876018 2461 tablet_bootstrap.cc:438] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: No bootstrap required, opened a new log 01:26:42.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.876356 2461 ts_tablet_manager.cc:1047] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.004s 01:26:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.877707 2461 raft_consensus.cc:340] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.878211 2461 raft_consensus.cc:366] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:26:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.878448 2461 raft_consensus.cc:705] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6eaa0da28e264c548dcf48b8ea8f690e, State: Initialized, Role: FOLLOWER 01:26:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.879007 2461 consensus_queue.cc:229] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.881366 2461 ts_tablet_manager.cc:1075] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s 01:26:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.903014 2482 raft_consensus.cc:472] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.903525 2482 raft_consensus.cc:494] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.905866 2482 leader_election.cc:296] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533), 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259) 01:26:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.919939 2108 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "02d12d6ae86c466fb511f0ddef2f6fc7" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" is_pre_election: true 01:26:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.920557 2108 raft_consensus.cc:2346] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 0. 01:26:42.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.922224 2305 leader_election.cc:310] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:42.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.923162 2482 raft_consensus.cc:472] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.923703 2482 raft_consensus.cc:494] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.924162 2240 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "02d12d6ae86c466fb511f0ddef2f6fc7" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" is_pre_election: true 01:26:42.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.925006 2240 raft_consensus.cc:2346] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 0. 01:26:42.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.926659 2482 leader_election.cc:296] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259), 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533) 01:26:42.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.927469 2493 raft_consensus.cc:2640] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Leader pre-election won for term 1 01:26:42.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.927898 2493 raft_consensus.cc:472] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:26:42.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.928191 2108 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8062ddd6695848d6b53496b0c2a33e88" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" is_pre_election: true 01:26:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.928417 2493 raft_consensus.cc:2886] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Advancing to term 1 01:26:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.928576 2240 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8062ddd6695848d6b53496b0c2a33e88" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" is_pre_election: true 01:26:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.928869 2108 raft_consensus.cc:2346] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 0. 01:26:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.929034 2240 raft_consensus.cc:2346] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 0. 01:26:42.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.930747 2301 leader_election.cc:310] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 134794f36230463ea5d0f3f800c08596, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:42.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:42.931190 2159 tablet.cc:1783] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:26:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.932986 2482 raft_consensus.cc:2640] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Leader pre-election won for term 1 01:26:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.933307 2482 raft_consensus.cc:472] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:26:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.933614 2482 raft_consensus.cc:2886] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Advancing to term 1 01:26:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.939815 2482 raft_consensus.cc:494] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.940136 2493 raft_consensus.cc:494] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.942179 2493 leader_election.cc:296] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 election: Requested vote from peers 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533), 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259) 01:26:42.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.943375 2108 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "02d12d6ae86c466fb511f0ddef2f6fc7" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" 01:26:42.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.943951 2108 raft_consensus.cc:2886] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Advancing to term 1 01:26:42.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.944846 2240 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8062ddd6695848d6b53496b0c2a33e88" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" 01:26:42.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.945916 2239 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "02d12d6ae86c466fb511f0ddef2f6fc7" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" 01:26:42.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.946385 2239 raft_consensus.cc:2886] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Advancing to term 1 01:26:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.947284 2493 raft_consensus.cc:472] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.947681 2493 raft_consensus.cc:494] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.948122 2107 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8062ddd6695848d6b53496b0c2a33e88" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" 01:26:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.948765 2107 raft_consensus.cc:2886] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Advancing to term 1 01:26:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.942242 2482 leader_election.cc:296] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 election: Requested vote from peers 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259), 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533) 01:26:42.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.947232 2240 raft_consensus.cc:2886] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Advancing to term 1 01:26:42.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.951797 2106 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "c0626d9f34c84a0488641522977ae3d0" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" is_pre_election: true 01:26:42.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.952211 2493 leader_election.cc:296] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533), 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259) 01:26:42.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.952580 2106 raft_consensus.cc:2346] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 0. 01:26:42.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.952620 2238 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "c0626d9f34c84a0488641522977ae3d0" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" is_pre_election: true 01:26:42.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.953363 2238 raft_consensus.cc:2346] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 0. 01:26:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.954918 2305 leader_election.cc:310] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.956118 2107 raft_consensus.cc:2346] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 1. 01:26:42.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.957245 2493 raft_consensus.cc:2640] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Leader pre-election won for term 1 01:26:42.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.957162 2108 raft_consensus.cc:2346] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 1. 01:26:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.958916 2305 leader_election.cc:310] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.959642 2482 raft_consensus.cc:2640] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Leader election won for term 1 01:26:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.959801 2240 raft_consensus.cc:2346] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 1. 01:26:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.960070 2305 leader_election.cc:310] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.960937 2239 raft_consensus.cc:2346] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 1. 01:26:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.957528 2493 raft_consensus.cc:472] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:26:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.963007 2493 raft_consensus.cc:2886] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Advancing to term 1 01:26:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.963984 2482 raft_consensus.cc:667] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 LEADER]: Becoming Leader. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Running, Role: LEADER 01:26:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.965971 2495 raft_consensus.cc:2640] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Leader election won for term 1 01:26:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.966853 2495 raft_consensus.cc:667] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 LEADER]: Becoming Leader. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Running, Role: LEADER 01:26:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.965939 2482 consensus_queue.cc:206] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.967672 2495 consensus_queue.cc:206] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } 01:26:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.971421 2493 raft_consensus.cc:494] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.975925 2493 leader_election.cc:296] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 election: Requested vote from peers 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65:51533), 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259) 01:26:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.975885 2108 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "c0626d9f34c84a0488641522977ae3d0" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" 01:26:42.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.976423 2108 raft_consensus.cc:2886] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Advancing to term 1 01:26:42.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.976758 2239 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "c0626d9f34c84a0488641522977ae3d0" candidate_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" 01:26:42.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.977365 2239 raft_consensus.cc:2886] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Advancing to term 1 01:26:42.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.983487 2108 raft_consensus.cc:2346] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 1. 01:26:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.984063 2239 raft_consensus.cc:2346] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9bbb727f40b43e2956d6f77e8505f0b in term 1. 01:26:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.985107 2305 leader_election.cc:310] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.986745 2493 raft_consensus.cc:2640] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Leader election won for term 1 01:26:42.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.987546 2493 raft_consensus.cc:667] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 LEADER]: Becoming Leader. State: Replica: b9bbb727f40b43e2956d6f77e8505f0b, State: Running, Role: LEADER 01:26:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.988350 2493 consensus_queue.cc:206] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:42.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:42.993232 1901 catalog_manager.cc:3791] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b reported cstate change: term changed from 0 to 1, leader changed from to b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67). New cstate: current_term: 1 leader_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } health_report { overall_health: HEALTHY } } } 01:26:43.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.018659 1901 catalog_manager.cc:3791] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b reported cstate change: term changed from 0 to 1, leader changed from to b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67). New cstate: current_term: 1 leader_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } health_report { overall_health: UNKNOWN } } } 01:26:43.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.019778 1901 catalog_manager.cc:3791] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b reported cstate change: term changed from 0 to 1, leader changed from to b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67). New cstate: current_term: 1 leader_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } health_report { overall_health: HEALTHY } } } 01:26:43.113 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@65828363@1703052131 returned Deferred@1534084728(state=PENDING, result=null, callback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478) -> (continuation of Deferred@1396756269 after org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245) -> (continuation of Deferred@657261008 after org.apache.kudu.client.AsyncKuduClient$15@4aaf9eb6@1253023414) -> (continuation of Deferred@820904281 after org.apache.kudu.client.AsyncKuduClient$15@65828363@1703052131), errback=(continuation of Deferred@1299770537 after org.apache.kudu.client.AsyncKuduClient$2@61ba754d@1639609677) -> (continuation of Deferred@1283689476 after org.apache.kudu.client.AsyncKuduClient$15@14648b60@342133600) -> (continuation of Deferred@1862630731 after org.apache.kudu.client.AsyncKuduClient$15@369fbe66@916438630) -> (continuation of Deferred@255430681 after org.apache.kudu.client.AsyncKuduClient$15@5f8ad440@1602933824) -> (continuation of Deferred@11903467 after org.apache.kudu.client.AsyncKuduClient$15@449e7316@1151234838) -> (continuation of Deferred@1257159962 after org.apache.kudu.client.AsyncKuduClient$15@9945d69@160718185) -> (continuation of Deferred@1355300155 after org.apache.kudu.client.AsyncKuduClient$15@7ddf6192@2111791506) -> (continuation of Deferred@1151674698 after org.apache.kudu.client.AsyncKuduClient$15@7b1c5d5e@2065456478) -> (continuation of Deferred@1396756269 after org.apache.kudu.client.AsyncKuduClient$15@427003bd@1114637245) -> (continuation of Deferred@657261008 after org.apache.kudu.client.AsyncKuduClient$15@4aaf9eb6@1253023414) -> (continuation of Deferred@820904281 after org.apache.kudu.client.AsyncKuduClient$15@65828363@1703052131)), so the following Deferred is getting paused: Deferred@820904281(state=PAUSED, result=Deferred@1534084728, callback=, errback=) 01:26:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.208377 2483 raft_consensus.cc:472] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:26:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.208848 2483 raft_consensus.cc:494] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.211014 2483 leader_election.cc:296] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259), b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179) 01:26:43.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.222000 2239 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "71e4dc0a850f4442b6c11bfcaa1b5ce8" candidate_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" is_pre_election: true 01:26:43.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.222479 2239 raft_consensus.cc:2346] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6eaa0da28e264c548dcf48b8ea8f690e in term 0. 01:26:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.223897 2021 leader_election.cc:310] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 134794f36230463ea5d0f3f800c08596, 6eaa0da28e264c548dcf48b8ea8f690e; no voters: 01:26:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.224189 2372 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "71e4dc0a850f4442b6c11bfcaa1b5ce8" candidate_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" is_pre_election: true 01:26:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.224802 2483 raft_consensus.cc:2640] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Leader pre-election won for term 1 01:26:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.224808 2372 raft_consensus.cc:2346] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6eaa0da28e264c548dcf48b8ea8f690e in term 0. 01:26:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.225026 2483 raft_consensus.cc:472] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:26:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.225318 2483 raft_consensus.cc:2886] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 0 FOLLOWER]: Advancing to term 1 01:26:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.228976 2483 raft_consensus.cc:494] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.230283 2483 leader_election.cc:296] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [CANDIDATE]: Term 1 election: Requested vote from peers 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259), b9bbb727f40b43e2956d6f77e8505f0b (127.1.193.67:36179) 01:26:43.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.232000 2239 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "71e4dc0a850f4442b6c11bfcaa1b5ce8" candidate_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "134794f36230463ea5d0f3f800c08596" 01:26:43.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.232054 2372 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "71e4dc0a850f4442b6c11bfcaa1b5ce8" candidate_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" 01:26:43.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.232493 2372 raft_consensus.cc:2886] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 0 FOLLOWER]: Advancing to term 1 01:26:43.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.232493 2239 raft_consensus.cc:2886] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 0 FOLLOWER]: Advancing to term 1 01:26:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.237898 2239 raft_consensus.cc:2346] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6eaa0da28e264c548dcf48b8ea8f690e in term 1. 01:26:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.238088 2372 raft_consensus.cc:2346] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6eaa0da28e264c548dcf48b8ea8f690e in term 1. 01:26:43.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.239490 2021 leader_election.cc:310] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6eaa0da28e264c548dcf48b8ea8f690e, b9bbb727f40b43e2956d6f77e8505f0b; no voters: 01:26:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.240128 2483 raft_consensus.cc:2640] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Leader election won for term 1 01:26:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.241789 2483 raft_consensus.cc:667] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 LEADER]: Becoming Leader. State: Replica: 6eaa0da28e264c548dcf48b8ea8f690e, State: Running, Role: LEADER 01:26:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.242478 2483 consensus_queue.cc:206] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } } 01:26:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.255575 1901 catalog_manager.cc:3791] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e reported cstate change: term changed from 0 to 1, leader changed from to 6eaa0da28e264c548dcf48b8ea8f690e (127.1.193.65). New cstate: current_term: 1 leader_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 } health_report { overall_health: UNKNOWN } } } 01:26:43.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.417378 2239 raft_consensus.cc:1181] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Refusing update from remote peer b9bbb727f40b43e2956d6f77e8505f0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.419204 2493 consensus_queue.cc:984] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s 01:26:43.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.437327 2108 raft_consensus.cc:1181] T 02d12d6ae86c466fb511f0ddef2f6fc7 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Refusing update from remote peer b9bbb727f40b43e2956d6f77e8505f0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.439913 2482 consensus_queue.cc:984] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:26:43.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.454222 2239 raft_consensus.cc:1181] T c0626d9f34c84a0488641522977ae3d0 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Refusing update from remote peer b9bbb727f40b43e2956d6f77e8505f0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.456416 2482 consensus_queue.cc:984] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:26:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.469025 2108 raft_consensus.cc:1181] T c0626d9f34c84a0488641522977ae3d0 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Refusing update from remote peer b9bbb727f40b43e2956d6f77e8505f0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.472116 2482 consensus_queue.cc:984] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s 01:26:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.477568 2107 raft_consensus.cc:1181] T 8062ddd6695848d6b53496b0c2a33e88 P 6eaa0da28e264c548dcf48b8ea8f690e [term 1 FOLLOWER]: Refusing update from remote peer b9bbb727f40b43e2956d6f77e8505f0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.480417 2482 consensus_queue.cc:984] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6eaa0da28e264c548dcf48b8ea8f690e" member_type: VOTER last_known_addr { host: "127.1.193.65" port: 51533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:26:43.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.503172 2239 raft_consensus.cc:1181] T 8062ddd6695848d6b53496b0c2a33e88 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Refusing update from remote peer b9bbb727f40b43e2956d6f77e8505f0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.506464 2522 consensus_queue.cc:984] T 8062ddd6695848d6b53496b0c2a33e88 P b9bbb727f40b43e2956d6f77e8505f0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:26:43.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.685194 2239 raft_consensus.cc:1181] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 134794f36230463ea5d0f3f800c08596 [term 1 FOLLOWER]: Refusing update from remote peer 6eaa0da28e264c548dcf48b8ea8f690e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.686952 2483 consensus_queue.cc:984] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [LEADER]: Connected to new peer: Peer: permanent_uuid: "134794f36230463ea5d0f3f800c08596" member_type: VOTER last_known_addr { host: "127.1.193.66" port: 39259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:26:43.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.706099 2372 raft_consensus.cc:1181] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P b9bbb727f40b43e2956d6f77e8505f0b [term 1 FOLLOWER]: Refusing update from remote peer 6eaa0da28e264c548dcf48b8ea8f690e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:26:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:43.716413 2483 consensus_queue.cc:984] T 71e4dc0a850f4442b6c11bfcaa1b5ce8 P 6eaa0da28e264c548dcf48b8ea8f690e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b9bbb727f40b43e2956d6f77e8505f0b" member_type: VOTER last_known_addr { host: "127.1.193.67" port: 36179 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:26:43.738 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:43.741 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:43.742 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:43.744 [DEBUG - New I/O worker #1] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299998, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299998, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299998, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:43.746 [DEBUG - New I/O worker #1] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=39), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.193.125:37435, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:43.754 [DEBUG - New I/O worker #1] (Deferred.java:1330) callback=retry RPC@1213737005 returned Deferred@1894225933(state=PENDING, result=null, callback=(continuation of Deferred@277340782 after retry RPC@1213737005), errback=(continuation of Deferred@277340782 after retry RPC@1213737005)), so the following Deferred is getting paused: Deferred@277340782(state=PAUSED, result=Deferred@1894225933, callback=org.apache.kudu.client.AsyncKuduSession$2@53702250 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ae06153 -> wakeup thread Time-limited test) 01:26:43.832 [DEBUG - New I/O worker #4] (Negotiator.java:764) Authenticated connection [id: 0xd61bad8e, /127.1.193.65:45539 => /127.1.193.65:51533] using TOKEN/null 01:26:43.926 [DEBUG - New I/O worker #5] (Negotiator.java:764) Authenticated connection [id: 0x2b7cdd0b, /127.1.193.67:53692 => /127.1.193.67:36179] using TOKEN/null 01:26:44.171 [DEBUG - pool-3-thread-2] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.204 [DEBUG - pool-3-thread-10] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.255 [DEBUG - pool-3-thread-16] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.215 [DEBUG - pool-3-thread-1] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.282 [DEBUG - pool-3-thread-8] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.278 [DEBUG - pool-3-thread-13] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.270 [DEBUG - New I/O worker #12] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.269 [DEBUG - pool-3-thread-6] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.268 [DEBUG - pool-3-thread-11] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.268 [DEBUG - New I/O worker #17] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.304 [DEBUG - New I/O worker #20] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.332 [DEBUG - pool-3-thread-7] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.415 [DEBUG - pool-3-thread-9] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.447 [DEBUG - pool-3-thread-18] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.450 [DEBUG - pool-3-thread-14] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.458 [DEBUG - pool-3-thread-3] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.460 [DEBUG - pool-3-thread-22] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.462 [DEBUG - New I/O worker #41] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.463 [DEBUG - New I/O worker #27] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.462 [DEBUG - New I/O worker #42] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.475 [DEBUG - New I/O worker #36] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.475 [DEBUG - New I/O worker #29] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.476 [DEBUG - New I/O worker #22] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.477 [DEBUG - pool-3-thread-15] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.478 [DEBUG - New I/O worker #12] (Negotiator.java:764) Authenticated connection [id: 0xfbc91d5f, /127.1.193.125:50071 => /127.1.193.125:37435] using SASL/PLAIN 01:26:44.493 [DEBUG - pool-3-thread-4] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.502 [DEBUG - pool-3-thread-5] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.508 [DEBUG - pool-3-thread-27] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.525 [DEBUG - pool-3-thread-17] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.562 [DEBUG - pool-3-thread-25] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.594 [DEBUG - New I/O worker #39] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.595 [DEBUG - New I/O worker #35] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.596 [DEBUG - New I/O worker #72] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.600 [DEBUG - New I/O worker #10] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.601 [DEBUG - New I/O worker #38] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.632 [DEBUG - pool-3-thread-35] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.637 [DEBUG - New I/O worker #101] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.666 [DEBUG - New I/O worker #56] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.683 [DEBUG - pool-3-thread-38] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.689 [DEBUG - pool-3-thread-23] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.692 [DEBUG - pool-3-thread-26] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.692 [DEBUG - New I/O worker #32] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.693 [DEBUG - New I/O worker #24] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.694 [DEBUG - New I/O worker #77] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.694 [DEBUG - New I/O worker #117] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.694 [DEBUG - New I/O worker #14] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.694 [DEBUG - pool-3-thread-19] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.695 [DEBUG - New I/O worker #16] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.695 [DEBUG - New I/O worker #18] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.696 [DEBUG - pool-3-thread-29] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.696 [DEBUG - New I/O worker #55] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.700 [DEBUG - New I/O worker #53] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.701 [DEBUG - New I/O worker #86] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.707 [DEBUG - pool-3-thread-12] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.720 [DEBUG - pool-3-thread-21] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.733 [DEBUG - pool-3-thread-39] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.735 [DEBUG - New I/O worker #11] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.744 [DEBUG - New I/O worker #75] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.748 [DEBUG - New I/O worker #13] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.755 [DEBUG - New I/O worker #68] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.758 [DEBUG - New I/O worker #15] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.760 [DEBUG - New I/O worker #106] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.762 [DEBUG - New I/O worker #34] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.764 [DEBUG - New I/O worker #61] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.764 [DEBUG - New I/O worker #48] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:44.765 [DEBUG - New I/O worker #62] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:44.766 [DEBUG - New I/O worker #19] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.767 [DEBUG - New I/O worker #46] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.768 [DEBUG - New I/O worker #42] (Negotiator.java:764) Authenticated connection [id: 0xdc9c45ca, /127.1.193.126:50819 => /127.1.193.126:40739] using SASL/PLAIN 01:26:44.764 [DEBUG - New I/O worker #21] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:44.772 [DEBUG - New I/O worker #35] (Negotiator.java:764) Authenticated connection [id: 0x4a9e3145, /127.1.193.124:46026 => /127.1.193.124:58247] using SASL/PLAIN 01:26:44.772 [DEBUG - New I/O worker #27] (Negotiator.java:764) Authenticated connection [id: 0x3d91967e, /127.1.193.125:50074 => /127.1.193.125:37435] using SASL/PLAIN 01:26:44.787 [DEBUG - New I/O worker #41] (Negotiator.java:764) Authenticated connection [id: 0x9f8acc67, /127.1.193.126:50816 => /127.1.193.126:40739] using SASL/PLAIN 01:26:44.806 [DEBUG - pool-3-thread-30] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.834 [DEBUG - pool-3-thread-33] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.854 [DEBUG - pool-3-thread-37] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:44.977 [DEBUG - pool-3-thread-32] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.108 [DEBUG - New I/O worker #17] (Negotiator.java:764) Authenticated connection [id: 0xea049efc, /127.1.193.124:45996 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.102 [DEBUG - New I/O worker #54] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.101 [DEBUG - New I/O worker #20] (Negotiator.java:764) Authenticated connection [id: 0x34933049, /127.1.193.126:50810 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.041 [DEBUG - New I/O worker #161] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.118 [DEBUG - pool-3-thread-24] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.136 [DEBUG - New I/O worker #149] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.138 [DEBUG - New I/O worker #52] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.138 [DEBUG - New I/O worker #23] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.138 [DEBUG - New I/O worker #80] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.138 [DEBUG - New I/O worker #159] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.138 [DEBUG - New I/O worker #116] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.146 [DEBUG - New I/O worker #15] (Negotiator.java:764) Authenticated connection [id: 0xeeb73400, /127.1.193.125:50116 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.147 [DEBUG - New I/O worker #50] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.148 [DEBUG - New I/O worker #143] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.150 [DEBUG - New I/O worker #13] (Negotiator.java:764) Authenticated connection [id: 0x3293a2cc, /127.1.193.125:50107 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.157 [DEBUG - New I/O worker #65] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.157 [DEBUG - New I/O worker #37] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.159 [DEBUG - New I/O worker #98] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.159 [DEBUG - New I/O worker #40] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.159 [DEBUG - New I/O worker #148] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.160 [DEBUG - New I/O worker #44] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.162 [DEBUG - New I/O worker #179] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.162 [DEBUG - New I/O worker #101] (Negotiator.java:764) Authenticated connection [id: 0xe0a8264c, /127.1.193.125:50122 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.168 [DEBUG - New I/O worker #110] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.171 [DEBUG - pool-3-thread-69] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.173 [DEBUG - pool-3-thread-36] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.174 [DEBUG - pool-3-thread-31] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.164 [DEBUG - New I/O worker #176] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.176 [DEBUG - pool-3-thread-20] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.168 [DEBUG - pool-3-thread-34] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.165 [DEBUG - New I/O worker #245] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.178 [DEBUG - pool-3-thread-28] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.165 [DEBUG - New I/O worker #74] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.164 [DEBUG - New I/O worker #26] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.164 [DEBUG - New I/O worker #217] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.164 [DEBUG - New I/O worker #73] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.171 [DEBUG - New I/O worker #25] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.210 [DEBUG - New I/O worker #34] (Negotiator.java:764) Authenticated connection [id: 0x7c89b17c, /127.1.193.124:46018 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.222 [DEBUG - New I/O worker #56] (Negotiator.java:764) Authenticated connection [id: 0x09959b02, /127.1.193.125:50128 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.229 [DEBUG - New I/O worker #106] (Negotiator.java:764) Authenticated connection [id: 0x57718290, /127.1.193.125:50113 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.228 [DEBUG - New I/O worker #217] (Negotiator.java:764) Authenticated connection [id: 0xde6442d8, /127.1.193.124:46056 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.225 [DEBUG - New I/O worker #73] (Negotiator.java:764) Authenticated connection [id: 0x5d8f30b5, /127.1.193.125:50155 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.231 [DEBUG - New I/O worker #159] (Negotiator.java:764) Authenticated connection [id: 0x32b89c7f, /127.1.193.124:46086 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.223 [DEBUG - New I/O worker #68] (Negotiator.java:764) Authenticated connection [id: 0xc274e3e9, /127.1.193.126:50849 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.245 [DEBUG - New I/O worker #23] (Negotiator.java:764) Authenticated connection [id: 0x63f5314f, /127.1.193.125:50149 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.247 [DEBUG - New I/O worker #11] (Negotiator.java:764) Authenticated connection [id: 0x7ac6a0d8, /127.1.193.125:50137 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.252 [DEBUG - New I/O worker #116] (Negotiator.java:764) Authenticated connection [id: 0x77af340d, /127.1.193.126:50882 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.256 [DEBUG - New I/O worker #117] (Negotiator.java:764) Authenticated connection [id: 0xfcf4c126, /127.1.193.126:50835 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.256 [DEBUG - New I/O worker #72] (Negotiator.java:764) Authenticated connection [id: 0x553b0121, /127.1.193.124:46014 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.264 [DEBUG - New I/O worker #179] (Negotiator.java:764) Authenticated connection [id: 0x822219ab, /127.1.193.124:46050 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.265 [DEBUG - New I/O worker #245] (Negotiator.java:764) Authenticated connection [id: 0x0a6ff7a1, /127.1.193.126:50885 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.269 [DEBUG - New I/O worker #98] (Negotiator.java:764) Authenticated connection [id: 0xe39aeced, /127.1.193.125:50134 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.277 [DEBUG - New I/O worker #75] (Negotiator.java:764) Authenticated connection [id: 0xf497b2a9, /127.1.193.125:50104 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.285 [DEBUG - New I/O worker #143] (Negotiator.java:764) Authenticated connection [id: 0x2e3f7352, /127.1.193.125:50152 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.296 [DEBUG - New I/O worker #38] (Negotiator.java:764) Authenticated connection [id: 0x0d279761, /127.1.193.126:50822 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.294 [DEBUG - New I/O worker #86] (Negotiator.java:764) Authenticated connection [id: 0x57bc8f99, /127.1.193.126:50838 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.293 [DEBUG - New I/O worker #10] (Negotiator.java:764) Authenticated connection [id: 0x68ec0e27, /127.1.193.125:50086 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.291 [DEBUG - New I/O worker #74] (Negotiator.java:764) Authenticated connection [id: 0x2c7bb827, /127.1.193.124:46065 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.291 [DEBUG - pool-3-thread-43] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.286 [DEBUG - New I/O worker #52] (Negotiator.java:764) Authenticated connection [id: 0xdec3b25e, /127.1.193.124:46044 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.304 [DEBUG - New I/O worker #176] (Negotiator.java:764) Authenticated connection [id: 0xb6fd4a37, /127.1.193.125:50158 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.374 [DEBUG - pool-3-thread-42] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.467 [DEBUG - New I/O worker #174] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.477 [DEBUG - New I/O worker #36] (Negotiator.java:764) Authenticated connection [id: 0x21437822, /127.1.193.124:46005 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.479 [DEBUG - New I/O worker #19] (Negotiator.java:764) Authenticated connection [id: 0xf9ceb9cd, /127.1.193.125:50093 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.480 [DEBUG - New I/O worker #29] (Negotiator.java:764) Authenticated connection [id: 0x08606a5e, /127.1.193.124:46002 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.482 [DEBUG - New I/O worker #16] (Negotiator.java:764) Authenticated connection [id: 0x5ea2561a, /127.1.193.125:50077 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.483 [DEBUG - New I/O worker #22] (Negotiator.java:764) Authenticated connection [id: 0x541d7116, /127.1.193.125:50080 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.477 [DEBUG - New I/O worker #39] (Negotiator.java:764) Authenticated connection [id: 0x5e763a9a, /127.1.193.124:45999 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.501 [DEBUG - New I/O worker #144] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.502 [DEBUG - New I/O worker #145] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.502 [DEBUG - New I/O worker #396] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.503 [DEBUG - New I/O worker #147] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.504 [DEBUG - New I/O worker #218] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.504 [DEBUG - New I/O worker #104] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.504 [DEBUG - New I/O worker #49] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.503 [DEBUG - New I/O worker #134] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.502 [DEBUG - New I/O worker #111] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.506 [DEBUG - New I/O worker #154] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.506 [DEBUG - New I/O worker #139] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.507 [DEBUG - New I/O worker #43] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.507 [DEBUG - New I/O worker #30] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.508 [DEBUG - New I/O worker #136] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.506 [DEBUG - New I/O worker #203] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.508 [DEBUG - New I/O worker #187] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.509 [DEBUG - New I/O worker #131] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.511 [DEBUG - New I/O worker #82] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.512 [DEBUG - New I/O worker #255] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.505 [DEBUG - New I/O worker #155] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.508 [DEBUG - New I/O worker #398] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.515 [DEBUG - New I/O worker #105] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.507 [DEBUG - New I/O worker #157] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.519 [DEBUG - New I/O worker #188] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.519 [DEBUG - New I/O worker #191] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.518 [DEBUG - New I/O worker #61] (Negotiator.java:764) Authenticated connection [id: 0x3614888c, /127.1.193.126:50832 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.517 [DEBUG - New I/O worker #137] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.517 [DEBUG - New I/O worker #181] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.517 [DEBUG - New I/O worker #184] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.517 [DEBUG - New I/O worker #153] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.516 [DEBUG - New I/O worker #146] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.516 [DEBUG - New I/O worker #263] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.516 [DEBUG - New I/O worker #185] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.516 [DEBUG - New I/O worker #109] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.516 [DEBUG - New I/O worker #130] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.526 [DEBUG - New I/O worker #92] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.516 [DEBUG - New I/O worker #138] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.516 [DEBUG - New I/O worker #102] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.516 [DEBUG - New I/O worker #60] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.537 [DEBUG - New I/O worker #182] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.538 [DEBUG - New I/O worker #257] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.516 [DEBUG - New I/O worker #141] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.540 [DEBUG - New I/O worker #192] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.540 [DEBUG - New I/O worker #150] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.540 [DEBUG - New I/O worker #225] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.516 [DEBUG - New I/O worker #325] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.516 [DEBUG - New I/O worker #199] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.515 [DEBUG - New I/O worker #273] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.515 [DEBUG - New I/O worker #238] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.515 [DEBUG - New I/O worker #76] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.515 [DEBUG - New I/O worker #175] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.515 [DEBUG - New I/O worker #186] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.543 [DEBUG - New I/O worker #228] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.541 [DEBUG - New I/O worker #51] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.534 [DEBUG - New I/O worker #28] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.533 [DEBUG - pool-3-thread-45] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.533 [DEBUG - New I/O worker #53] (Negotiator.java:764) Authenticated connection [id: 0x56a909aa, /127.1.193.126:50813 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.561 [DEBUG - New I/O worker #78] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.561 [DEBUG - New I/O worker #71] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.565 [DEBUG - New I/O worker #222] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.576 [DEBUG - New I/O worker #312] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.577 [DEBUG - New I/O worker #332] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.577 [DEBUG - New I/O worker #399] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.585 [DEBUG - New I/O worker #229] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.602 [DEBUG - New I/O worker #139] (Negotiator.java:764) Authenticated connection [id: 0xdb8a0d68, /127.1.193.125:50164 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.650 [DEBUG - pool-3-thread-44] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.655 [DEBUG - pool-3-thread-40] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.662 [DEBUG - pool-3-thread-68] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.680 [DEBUG - New I/O worker #394] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.684 [DEBUG - New I/O worker #502] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.680 [DEBUG - New I/O worker #480] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.697 [DEBUG - New I/O worker #216] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.700 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.700 [DEBUG - New I/O worker #256] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.704 [DEBUG - New I/O worker #302] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.707 [DEBUG - New I/O worker #258] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.711 [DEBUG - New I/O worker #309] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.717 [DEBUG - pool-3-thread-50] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.717 [DEBUG - New I/O worker #343] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.741 [DEBUG - New I/O worker #310] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:45.754 [DEBUG - New I/O worker #303] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:45.839 [DEBUG - New I/O worker #342] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:45.847 [DEBUG - New I/O worker #62] (Negotiator.java:764) Authenticated connection [id: 0xcd1e07e7, /127.1.193.124:46024 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.849 [DEBUG - New I/O worker #54] (Negotiator.java:764) Authenticated connection [id: 0x815444fe, /127.1.193.125:50096 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.850 [DEBUG - New I/O worker #26] (Negotiator.java:764) Authenticated connection [id: 0xaa2bb1d1, /127.1.193.125:50110 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.850 [DEBUG - New I/O worker #14] (Negotiator.java:764) Authenticated connection [id: 0x3264d528, /127.1.193.124:46011 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.851 [DEBUG - New I/O worker #48] (Negotiator.java:764) Authenticated connection [id: 0x21725be1, /127.1.193.126:50840 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.852 [DEBUG - New I/O worker #21] (Negotiator.java:764) Authenticated connection [id: 0x8473a571, /127.1.193.125:50092 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.855 [DEBUG - New I/O worker #80] (Negotiator.java:764) Authenticated connection [id: 0x7ed7f077, /127.1.193.124:46021 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.856 [DEBUG - New I/O worker #161] (Negotiator.java:764) Authenticated connection [id: 0x29010fd7, /127.1.193.126:50843 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.856 [DEBUG - New I/O worker #46] (Negotiator.java:764) Authenticated connection [id: 0x53dc5778, /127.1.193.125:50099 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.859 [DEBUG - New I/O worker #55] (Negotiator.java:764) Authenticated connection [id: 0x12bca1e3, /127.1.193.125:50089 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.861 [DEBUG - New I/O worker #148] (Negotiator.java:764) Authenticated connection [id: 0xd78b6e39, /127.1.193.124:46038 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.866 [DEBUG - New I/O worker #65] (Negotiator.java:764) Authenticated connection [id: 0x8f4655fb, /127.1.193.126:50846 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.866 [DEBUG - New I/O worker #50] (Negotiator.java:764) Authenticated connection [id: 0x23347914, /127.1.193.126:50855 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.868 [DEBUG - New I/O worker #32] (Negotiator.java:764) Authenticated connection [id: 0xa2897b44, /127.1.193.126:50825 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.868 [DEBUG - New I/O worker #144] (Negotiator.java:764) Authenticated connection [id: 0x23b77451, /127.1.193.124:46047 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.866 [DEBUG - New I/O worker #25] (Negotiator.java:764) Authenticated connection [id: 0xe9f0d96b, /127.1.193.124:46041 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.869 [DEBUG - New I/O worker #82] (Negotiator.java:764) Authenticated connection [id: 0x48ecc2e3, /127.1.193.124:46053 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.877 [DEBUG - New I/O worker #149] (Negotiator.java:764) Authenticated connection [id: 0x9e11c1ab, /127.1.193.126:50852 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.886 [DEBUG - New I/O worker #40] (Negotiator.java:764) Authenticated connection [id: 0x88d2eeb1, /127.1.193.124:46035 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.887 [DEBUG - New I/O worker #43] (Negotiator.java:764) Authenticated connection [id: 0x3011ab80, /127.1.193.126:50876 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.895 [DEBUG - New I/O worker #77] (Negotiator.java:764) Authenticated connection [id: 0xdd2e3b80, /127.1.193.126:50828 => /127.1.193.126:40739] using SASL/PLAIN 01:26:45.900 [DEBUG - New I/O worker #44] (Negotiator.java:764) Authenticated connection [id: 0x2b2d3011, /127.1.193.124:46032 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.905 [DEBUG - New I/O worker #111] (Negotiator.java:764) Authenticated connection [id: 0x94a6597c, /127.1.193.125:50182 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.911 [DEBUG - New I/O worker #174] (Negotiator.java:764) Authenticated connection [id: 0x84a5d338, /127.1.193.125:50146 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.913 [DEBUG - New I/O worker #110] (Negotiator.java:764) Authenticated connection [id: 0x62ba3e4b, /127.1.193.124:46029 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.914 [DEBUG - pool-3-thread-74] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:45.914 [DEBUG - New I/O worker #187] (Negotiator.java:764) Authenticated connection [id: 0x53968bf8, /127.1.193.124:46116 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.915 [DEBUG - New I/O worker #18] (Negotiator.java:764) Authenticated connection [id: 0x987f7e04, /127.1.193.125:50083 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.915 [DEBUG - New I/O worker #396] (Negotiator.java:764) Authenticated connection [id: 0xfb7446b5, /127.1.193.125:50194 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.927 [DEBUG - New I/O worker #37] (Negotiator.java:764) Authenticated connection [id: 0x8a3ca46e, /127.1.193.125:50119 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.937 [DEBUG - New I/O worker #394] (Negotiator.java:764) Authenticated connection [id: 0xd6a8ea56, /127.1.193.125:50197 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.946 [DEBUG - New I/O worker #145] (Negotiator.java:764) Authenticated connection [id: 0x516822a1, /127.1.193.125:50173 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.948 [DEBUG - New I/O worker #258] (Negotiator.java:764) Authenticated connection [id: 0x9d8ad078, /127.1.193.125:50203 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.951 [DEBUG - New I/O worker #263] (Negotiator.java:764) Authenticated connection [id: 0xa02e43d9, /127.1.193.124:46095 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.956 [DEBUG - New I/O worker #216] (Negotiator.java:764) Authenticated connection [id: 0xf0ae2b3c, /127.1.193.125:50200 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.960 [DEBUG - New I/O worker #325] (Negotiator.java:764) Authenticated connection [id: 0x641cace3, /127.1.193.124:46092 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.963 [DEBUG - New I/O worker #134] (Negotiator.java:764) Authenticated connection [id: 0x9869b1ea, /127.1.193.125:50176 => /127.1.193.125:37435] using SASL/PLAIN 01:26:45.970 [DEBUG - New I/O worker #30] (Negotiator.java:764) Authenticated connection [id: 0x8f7e9335, /127.1.193.124:46074 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.971 [DEBUG - New I/O worker #146] (Negotiator.java:764) Authenticated connection [id: 0xd9680fe1, /127.1.193.124:46113 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.972 [DEBUG - New I/O worker #155] (Negotiator.java:764) Authenticated connection [id: 0x9e897171, /127.1.193.124:46077 => /127.1.193.124:58247] using SASL/PLAIN 01:26:45.976 [DEBUG - New I/O worker #141] (Negotiator.java:764) Authenticated connection [id: 0xf34a11d4, /127.1.193.124:46059 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.099 [DEBUG - New I/O worker #310] (Negotiator.java:764) Authenticated connection [id: 0xbb5f172c, /127.1.193.124:46131 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.097 [DEBUG - New I/O worker #24] (Negotiator.java:764) Authenticated connection [id: 0xf497f19c, /127.1.193.124:46008 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.083 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.101 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1598), Traces: [0ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [63ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [648ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1394ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.101 [DEBUG - New I/O worker #49] (Negotiator.java:764) Authenticated connection [id: 0xf45b6f0f, /127.1.193.125:50179 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.083 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.288 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1983), Traces: [0ms] refreshing cache from master, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [497ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1193ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.083 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.289 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2114), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [316ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [949ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.095 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.349 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1871), Traces: [0ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [826ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1453ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.095 [DEBUG - New I/O worker #383] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.095 [DEBUG - New I/O worker #513] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.095 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.095 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.460 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2155), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [998ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1618ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1816ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.095 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.461 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1727), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [555ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.094 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.507 [DEBUG - New I/O worker #174] (Deferred.java:1330) callback=retry RPC@1288758354 returned Deferred@1223649552(state=PENDING, result=null, callback=(continuation of Deferred@233696873 after retry RPC@1288758354), errback=(continuation of Deferred@233696873 after retry RPC@1288758354)), so the following Deferred is getting paused: Deferred@233696873(state=PAUSED, result=Deferred@1223649552, callback=org.apache.kudu.client.AsyncKuduClient$6@7793e9b6 -> wakeup thread pool-3-thread-39, errback=passthrough -> wakeup thread pool-3-thread-39) 01:26:46.519 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.094 [DEBUG - New I/O worker #469] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.700 [DEBUG - pool-3-thread-65] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.701 [DEBUG - pool-3-thread-76] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.702 [DEBUG - pool-3-thread-58] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.703 [DEBUG - pool-3-thread-62] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.094 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.704 [DEBUG - pool-3-thread-63] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.704 [DEBUG - pool-3-thread-60] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.704 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2395), Traces: [0ms] refreshing cache from master, [133ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [468ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1570ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1619ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.094 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.705 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2254), Traces: [0ms] refreshing cache from master, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [850ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1428ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1433ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.094 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.726 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2278), Traces: [0ms] refreshing cache from master, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [811ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1415ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1460ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.094 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.093 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.093 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.749 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2332), Traces: [0ms] refreshing cache from master, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [819ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1488ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.749 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2446), Traces: [0ms] refreshing cache from master, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [974ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1569ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1625ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.093 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.092 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.092 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.783 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2520), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [530ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1237ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.092 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.092 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.787 [DEBUG - New I/O worker #16] (Deferred.java:1330) callback=retry RPC@289349017 returned Deferred@1222470139(state=PENDING, result=null, callback=(continuation of Deferred@1090183193 after retry RPC@289349017), errback=(continuation of Deferred@1090183193 after retry RPC@289349017)), so the following Deferred is getting paused: Deferred@1090183193(state=PAUSED, result=Deferred@1222470139, callback=org.apache.kudu.client.AsyncKuduClient$6@19d23c31 -> wakeup thread pool-3-thread-10, errback=passthrough -> wakeup thread pool-3-thread-10) 01:26:46.787 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2274), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1405ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.092 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.793 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1937), Traces: [0ms] refreshing cache from master, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [452ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.092 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.091 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.091 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.807 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2345), Traces: [0ms] refreshing cache from master, [17ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [834ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1432ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1456ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.090 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.811 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2103), Traces: [0ms] refreshing cache from master, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1272ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.090 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.824 [DEBUG - New I/O worker #23] (Deferred.java:1330) callback=retry RPC@86057316 returned Deferred@1076084644(state=PENDING, result=null, callback=(continuation of Deferred@91210846 after retry RPC@86057316), errback=(continuation of Deferred@91210846 after retry RPC@86057316)), so the following Deferred is getting paused: Deferred@91210846(state=PAUSED, result=Deferred@1076084644, callback=org.apache.kudu.client.AsyncKuduClient$6@71a93b21 -> wakeup thread pool-3-thread-12, errback=passthrough -> wakeup thread pool-3-thread-12) 01:26:46.090 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.831 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2141), Traces: [0ms] refreshing cache from master, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [545ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.831 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.090 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.090 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.852 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2326), Traces: [0ms] refreshing cache from master, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [707ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.089 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.866 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2371), Traces: [0ms] refreshing cache from master, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.865 [DEBUG - New I/O worker #56] (Deferred.java:1330) callback=retry RPC@1195818413 returned Deferred@760716916(state=PENDING, result=null, callback=(continuation of Deferred@1993038671 after retry RPC@1195818413), errback=(continuation of Deferred@1993038671 after retry RPC@1195818413)), so the following Deferred is getting paused: Deferred@1993038671(state=PAUSED, result=Deferred@760716916, callback=org.apache.kudu.client.AsyncKuduClient$6@5530263c -> wakeup thread pool-3-thread-17, errback=passthrough -> wakeup thread pool-3-thread-17) 01:26:46.859 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:46.852 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2289), Traces: [0ms] refreshing cache from master, [71ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [71ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [71ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [613ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.889 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.851 [DEBUG - New I/O worker #73] (Deferred.java:1330) callback=retry RPC@1366539708 returned Deferred@1431053130(state=PENDING, result=null, callback=(continuation of Deferred@273594247 after retry RPC@1366539708), errback=(continuation of Deferred@273594247 after retry RPC@1366539708)), so the following Deferred is getting paused: Deferred@273594247(state=PAUSED, result=Deferred@1431053130, callback=org.apache.kudu.client.AsyncKuduClient$6@6d46c82c -> wakeup thread pool-3-thread-23, errback=passthrough -> wakeup thread pool-3-thread-23) 01:26:46.824 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2365), Traces: [0ms] refreshing cache from master, [247ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [247ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [248ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [793ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1470ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.890 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.890 [DEBUG - New I/O worker #11] (Deferred.java:1330) callback=retry RPC@1569056442 returned Deferred@411530094(state=PENDING, result=null, callback=(continuation of Deferred@590327401 after retry RPC@1569056442), errback=(continuation of Deferred@590327401 after retry RPC@1569056442)), so the following Deferred is getting paused: Deferred@590327401(state=PAUSED, result=Deferred@411530094, callback=org.apache.kudu.client.AsyncKuduClient$6@29550b88 -> wakeup thread pool-3-thread-3, errback=passthrough -> wakeup thread pool-3-thread-3) 01:26:46.810 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2113), Traces: [0ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.891 [DEBUG - New I/O worker #143] (Deferred.java:1330) callback=retry RPC@959196544 returned Deferred@1024704125(state=PENDING, result=null, callback=(continuation of Deferred@2014390203 after retry RPC@959196544), errback=(continuation of Deferred@2014390203 after retry RPC@959196544)), so the following Deferred is getting paused: Deferred@2014390203(state=PAUSED, result=Deferred@1024704125, callback=org.apache.kudu.client.AsyncKuduClient$6@113bc609 -> wakeup thread pool-3-thread-29, errback=passthrough -> wakeup thread pool-3-thread-29) 01:26:46.810 [DEBUG - New I/O worker #75] (Deferred.java:1330) callback=retry RPC@1336108104 returned Deferred@62288996(state=PENDING, result=null, callback=(continuation of Deferred@2116448151 after retry RPC@1336108104), errback=(continuation of Deferred@2116448151 after retry RPC@1336108104)), so the following Deferred is getting paused: Deferred@2116448151(state=PAUSED, result=Deferred@62288996, callback=org.apache.kudu.client.AsyncKuduClient$6@7bd92110 -> wakeup thread pool-3-thread-22, errback=passthrough -> wakeup thread pool-3-thread-22) 01:26:46.892 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.807 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2114), Traces: [0ms] refreshing cache from master, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1412ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:46.893 [DEBUG - New I/O worker #98] (Deferred.java:1330) callback=retry RPC@632845799 returned Deferred@1541629960(state=PENDING, result=null, callback=(continuation of Deferred@349673271 after retry RPC@632845799), errback=(continuation of Deferred@349673271 after retry RPC@632845799)), so the following Deferred is getting paused: Deferred@349673271(state=PAUSED, result=Deferred@1541629960, callback=org.apache.kudu.client.AsyncKuduClient$6@2366d46e -> wakeup thread pool-3-thread-26, errback=passthrough -> wakeup thread pool-3-thread-26) 01:26:46.802 [DEBUG - New I/O worker #176] (Deferred.java:1330) callback=retry RPC@1516969822 returned Deferred@1582600395(state=PENDING, result=null, callback=(continuation of Deferred@458626405 after retry RPC@1516969822), errback=(continuation of Deferred@458626405 after retry RPC@1516969822)), so the following Deferred is getting paused: Deferred@458626405(state=PAUSED, result=Deferred@1582600395, callback=org.apache.kudu.client.AsyncKuduClient$6@28944497 -> wakeup thread pool-3-thread-37, errback=passthrough -> wakeup thread pool-3-thread-37) 01:26:46.793 [DEBUG - New I/O worker #106] (Deferred.java:1330) callback=retry RPC@1060287140 returned Deferred@1200117256(state=PENDING, result=null, callback=(continuation of Deferred@1206147408 after retry RPC@1060287140), errback=(continuation of Deferred@1206147408 after retry RPC@1060287140)), so the following Deferred is getting paused: Deferred@1206147408(state=PAUSED, result=Deferred@1200117256, callback=org.apache.kudu.client.AsyncKuduClient$6@169b1c0f -> wakeup thread pool-3-thread-27, errback=passthrough -> wakeup thread pool-3-thread-27) 01:26:46.899 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.793 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.787 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2483), Traces: [0ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [56ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1571ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1595ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.783 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2474), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [521ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1182ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.780 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1972), Traces: [0ms] refreshing cache from master, [311ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [312ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [312ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [817ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.914 [DEBUG - New I/O worker #22] (Deferred.java:1330) callback=retry RPC@1057710892 returned Deferred@448200420(state=PENDING, result=null, callback=(continuation of Deferred@1860219641 after retry RPC@1057710892), errback=(continuation of Deferred@1860219641 after retry RPC@1057710892)), so the following Deferred is getting paused: Deferred@1860219641(state=PAUSED, result=Deferred@448200420, callback=org.apache.kudu.client.AsyncKuduClient$6@48a8379e -> wakeup thread pool-3-thread-11, errback=passthrough -> wakeup thread pool-3-thread-11) 01:26:46.914 [DEBUG - New I/O worker #139] (Deferred.java:1330) callback=retry RPC@1239231490 returned Deferred@215984989(state=PENDING, result=null, callback=(continuation of Deferred@1697145699 after retry RPC@1239231490), errback=(continuation of Deferred@1697145699 after retry RPC@1239231490)), so the following Deferred is getting paused: Deferred@1697145699(state=PAUSED, result=Deferred@215984989, callback=org.apache.kudu.client.AsyncKuduClient$6@3d990731 -> wakeup thread pool-3-thread-30, errback=passthrough -> wakeup thread pool-3-thread-30) 01:26:46.769 [DEBUG - New I/O worker #55] (Deferred.java:1330) callback=retry RPC@793635359 returned Deferred@2008078518(state=PENDING, result=null, callback=(continuation of Deferred@514353060 after retry RPC@793635359), errback=(continuation of Deferred@514353060 after retry RPC@793635359)), so the following Deferred is getting paused: Deferred@514353060(state=PAUSED, result=Deferred@2008078518, callback=org.apache.kudu.client.AsyncKuduClient$6@fbf0baa -> wakeup thread pool-3-thread-16, errback=passthrough -> wakeup thread pool-3-thread-16) 01:26:46.915 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.768 [DEBUG - pool-3-thread-46] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.762 [DEBUG - New I/O worker #526] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.925 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:46.925 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:46.926 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:46.926 [DEBUG - New I/O worker #55] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:46.762 [DEBUG - New I/O worker #436] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.760 [DEBUG - New I/O worker #509] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.754 [DEBUG - New I/O worker #400] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.940 [DEBUG - New I/O worker #418] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.754 [DEBUG - New I/O worker #537] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.754 [DEBUG - New I/O worker #485] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.753 [DEBUG - New I/O worker #492] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.753 [DEBUG - New I/O worker #26] (Deferred.java:1330) callback=retry RPC@1662515174 returned Deferred@638936544(state=PENDING, result=null, callback=(continuation of Deferred@856381080 after retry RPC@1662515174), errback=(continuation of Deferred@856381080 after retry RPC@1662515174)), so the following Deferred is getting paused: Deferred@856381080(state=PAUSED, result=Deferred@638936544, callback=org.apache.kudu.client.AsyncKuduClient$6@17f9a195 -> wakeup thread pool-3-thread-9, errback=passthrough -> wakeup thread pool-3-thread-9) 01:26:46.945 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.753 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2419), Traces: [0ms] refreshing cache from master, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1255ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:46.946 [DEBUG - New I/O worker #19] (Deferred.java:1330) callback=retry RPC@1544688622 returned Deferred@1225587850(state=PENDING, result=null, callback=(continuation of Deferred@219857844 after retry RPC@1544688622), errback=(continuation of Deferred@219857844 after retry RPC@1544688622)), so the following Deferred is getting paused: Deferred@219857844(state=PAUSED, result=Deferred@1225587850, callback=org.apache.kudu.client.AsyncKuduClient$6@300ff7ba -> wakeup thread pool-3-thread-7, errback=passthrough -> wakeup thread pool-3-thread-7) 01:26:46.744 [DEBUG - New I/O worker #54] (Deferred.java:1330) callback=retry RPC@2018658863 returned Deferred@1028709185(state=PENDING, result=null, callback=(continuation of Deferred@517191021 after retry RPC@2018658863), errback=(continuation of Deferred@517191021 after retry RPC@2018658863)), so the following Deferred is getting paused: Deferred@517191021(state=PAUSED, result=Deferred@1028709185, callback=org.apache.kudu.client.AsyncKuduClient$6@ca741cc -> wakeup thread pool-3-thread-18, errback=passthrough -> wakeup thread pool-3-thread-18) 01:26:46.947 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.738 [DEBUG - New I/O worker #417] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.738 [DEBUG - New I/O worker #413] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.737 [DEBUG - New I/O worker #407] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.736 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:46.736 [DEBUG - New I/O worker #424] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.950 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:46.951 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:46.951 [DEBUG - New I/O worker #174] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:46.951 [DEBUG - New I/O worker #174] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:46.732 [DEBUG - New I/O worker #501] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.731 [DEBUG - New I/O worker #403] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.731 [DEBUG - New I/O worker #427] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.731 [DEBUG - New I/O worker #483] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.725 [DEBUG - New I/O worker #46] (Deferred.java:1330) callback=retry RPC@1882124285 returned Deferred@892104604(state=PENDING, result=null, callback=(continuation of Deferred@2073065304 after retry RPC@1882124285), errback=(continuation of Deferred@2073065304 after retry RPC@1882124285)), so the following Deferred is getting paused: Deferred@2073065304(state=PAUSED, result=Deferred@892104604, callback=org.apache.kudu.client.AsyncKuduClient$6@45319d0 -> wakeup thread pool-3-thread-14, errback=passthrough -> wakeup thread pool-3-thread-14) 01:26:46.725 [DEBUG - New I/O worker #21] (Deferred.java:1330) callback=retry RPC@2015009639 returned Deferred@1024985600(state=PENDING, result=null, callback=(continuation of Deferred@1478440956 after retry RPC@2015009639), errback=(continuation of Deferred@1478440956 after retry RPC@2015009639)), so the following Deferred is getting paused: Deferred@1478440956(state=PAUSED, result=Deferred@1024985600, callback=org.apache.kudu.client.AsyncKuduClient$6@cefd01c -> wakeup thread pool-3-thread-6, errback=passthrough -> wakeup thread pool-3-thread-6) 01:26:46.966 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.720 [DEBUG - New I/O worker #490] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.967 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.718 [DEBUG - New I/O worker #411] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.968 [DEBUG - New I/O worker #400] (Negotiator.java:764) Authenticated connection [id: 0x6f60356a, /127.1.193.125:50222 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.718 [DEBUG - New I/O worker #542] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.713 [DEBUG - New I/O worker #469] (Negotiator.java:764) Authenticated connection [id: 0x944513e7, /127.1.193.124:46134 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.706 [DEBUG - pool-3-thread-55] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.706 [DEBUG - pool-3-thread-77] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.705 [DEBUG - pool-3-thread-75] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.520 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1334), Traces: [0ms] refreshing cache from master, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.013 [DEBUG - New I/O worker #111] (Deferred.java:1330) callback=retry RPC@1295989320 returned Deferred@1224783671(state=PENDING, result=null, callback=(continuation of Deferred@201470067 after retry RPC@1295989320), errback=(continuation of Deferred@201470067 after retry RPC@1295989320)), so the following Deferred is getting paused: Deferred@201470067(state=PAUSED, result=Deferred@1224783671, callback=org.apache.kudu.client.AsyncKuduClient$6@91dcf50 -> wakeup thread pool-3-thread-36, errback=passthrough -> wakeup thread pool-3-thread-36) 01:26:46.510 [DEBUG - pool-3-thread-71] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.508 [DEBUG - pool-3-thread-79] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.507 [DEBUG - New I/O worker #18] (Deferred.java:1330) callback=retry RPC@216848457 returned Deferred@1240352688(state=PENDING, result=null, callback=(continuation of Deferred@1791011023 after retry RPC@216848457), errback=(continuation of Deferred@1791011023 after retry RPC@216848457)), so the following Deferred is getting paused: Deferred@1791011023(state=PAUSED, result=Deferred@1240352688, callback=org.apache.kudu.client.AsyncKuduClient$6@786d0f09 -> wakeup thread pool-3-thread-8, errback=passthrough -> wakeup thread pool-3-thread-8) 01:26:47.015 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.492 [DEBUG - pool-3-thread-80] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.492 [DEBUG - pool-3-thread-66] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.486 [DEBUG - pool-3-thread-61] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.484 [DEBUG - pool-3-thread-72] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:47.018 [DEBUG - New I/O worker #483] (Negotiator.java:764) Authenticated connection [id: 0x519782f6, /127.1.193.124:46147 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.472 [DEBUG - pool-3-thread-48] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.472 [DEBUG - New I/O worker #513] (Negotiator.java:764) Authenticated connection [id: 0x574d4548, /127.1.193.126:50948 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.460 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1288), Traces: [0ms] refreshing cache from master, [303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [751ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.020 [DEBUG - New I/O worker #396] (Deferred.java:1330) callback=retry RPC@2017961348 returned Deferred@2088250964(state=PENDING, result=null, callback=(continuation of Deferred@964341695 after retry RPC@2017961348), errback=(continuation of Deferred@964341695 after retry RPC@2017961348)), so the following Deferred is getting paused: Deferred@964341695(state=PAUSED, result=Deferred@2088250964, callback=org.apache.kudu.client.AsyncKuduClient$6@4d2764d1 -> wakeup thread pool-3-thread-69, errback=passthrough -> wakeup thread pool-3-thread-69) 01:26:46.453 [DEBUG - pool-3-thread-54] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.438 [DEBUG - New I/O worker #383] (Negotiator.java:764) Authenticated connection [id: 0x078d807f, /127.1.193.125:50209 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.437 [DEBUG - pool-3-thread-49] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.419 [DEBUG - pool-3-thread-78] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.376 [DEBUG - pool-3-thread-73] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.374 [DEBUG - pool-3-thread-41] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.373 [DEBUG - pool-3-thread-59] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:47.025 [DEBUG - New I/O worker #411] (Negotiator.java:764) Authenticated connection [id: 0x78bd119d, /127.1.193.125:50219 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.354 [DEBUG - New I/O worker #37] (Deferred.java:1330) callback=retry RPC@1444835161 returned Deferred@1377956162(state=PENDING, result=null, callback=(continuation of Deferred@388195682 after retry RPC@1444835161), errback=(continuation of Deferred@388195682 after retry RPC@1444835161)), so the following Deferred is getting paused: Deferred@388195682(state=PAUSED, result=Deferred@1377956162, callback=org.apache.kudu.client.AsyncKuduClient$6@3bf75be -> wakeup thread pool-3-thread-15, errback=passthrough -> wakeup thread pool-3-thread-15) 01:26:47.025 [DEBUG - New I/O worker #418] (Negotiator.java:764) Authenticated connection [id: 0x61d6b845, /127.1.193.124:46162 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.026 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.026 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.348 [DEBUG - New I/O worker #12] (Deferred.java:1330) callback=retry RPC@1577588687 returned Deferred@1513588950(state=PENDING, result=null, callback=(continuation of Deferred@523574772 after retry RPC@1577588687), errback=(continuation of Deferred@523574772 after retry RPC@1577588687)), so the following Deferred is getting paused: Deferred@523574772(state=PAUSED, result=Deferred@1513588950, callback=org.apache.kudu.client.AsyncKuduClient$6@30c80f67 -> wakeup thread pool-3-thread-2, errback=passthrough -> wakeup thread pool-3-thread-2) 01:26:46.348 [DEBUG - New I/O worker #27] (Deferred.java:1330) callback=retry RPC@158003396 returned Deferred@223704369(state=PENDING, result=null, callback=(continuation of Deferred@1213725439 after retry RPC@158003396), errback=(continuation of Deferred@1213725439 after retry RPC@158003396)), so the following Deferred is getting paused: Deferred@1213725439(state=PAUSED, result=Deferred@223704369, callback=org.apache.kudu.client.AsyncKuduClient$6@27e7baa2 -> wakeup thread pool-3-thread-13, errback=passthrough -> wakeup thread pool-3-thread-13) 01:26:47.027 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.027 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.288 [DEBUG - New I/O worker #15] (Deferred.java:1330) callback=retry RPC@387854617 returned Deferred@320976674(state=PENDING, result=null, callback=(continuation of Deferred@1445149474 after retry RPC@387854617), errback=(continuation of Deferred@1445149474 after retry RPC@387854617)), so the following Deferred is getting paused: Deferred@1445149474(state=PAUSED, result=Deferred@320976674, callback=org.apache.kudu.client.AsyncKuduClient$6@14f46f8 -> wakeup thread pool-3-thread-5, errback=passthrough -> wakeup thread pool-3-thread-5) 01:26:46.337 [DEBUG - pool-3-thread-53] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:47.028 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.028 [DEBUG - New I/O worker #501] (Negotiator.java:764) Authenticated connection [id: 0x6d696cc9, /127.1.193.126:50964 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.334 [DEBUG - pool-3-thread-64] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.333 [DEBUG - pool-3-thread-56] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.331 [DEBUG - pool-3-thread-67] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.311 [DEBUG - pool-3-thread-57] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.311 [DEBUG - New I/O worker #229] (Negotiator.java:764) Authenticated connection [id: 0x20bdaa1a, /127.1.193.126:50930 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.309 [DEBUG - New I/O worker #60] (Negotiator.java:764) Authenticated connection [id: 0x4b0af68e, /127.1.193.126:50888 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.308 [DEBUG - New I/O worker #182] (Negotiator.java:764) Authenticated connection [id: 0x521a3204, /127.1.193.126:50921 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.308 [DEBUG - New I/O worker #302] (Negotiator.java:764) Authenticated connection [id: 0xd39e153f, /127.1.193.126:50939 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.308 [DEBUG - New I/O worker #343] (Negotiator.java:764) Authenticated connection [id: 0xd8626147, /127.1.193.126:50942 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.306 [DEBUG - New I/O worker #309] (Negotiator.java:764) Authenticated connection [id: 0x7a7d1e38, /127.1.193.124:46128 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.306 [DEBUG - New I/O worker #175] (Negotiator.java:764) Authenticated connection [id: 0x7dd5908b, /127.1.193.125:50131 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.305 [DEBUG - New I/O worker #332] (Negotiator.java:764) Authenticated connection [id: 0x57c068a1, /127.1.193.126:50906 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.305 [DEBUG - New I/O worker #138] (Negotiator.java:764) Authenticated connection [id: 0xfff14577, /127.1.193.125:50185 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.305 [DEBUG - New I/O worker #188] (Negotiator.java:764) Authenticated connection [id: 0x2444bec6, /127.1.193.126:50891 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.304 [DEBUG - New I/O worker #312] (Negotiator.java:764) Authenticated connection [id: 0xc74da3e1, /127.1.193.126:50909 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.304 [DEBUG - New I/O worker #150] (Negotiator.java:764) Authenticated connection [id: 0xaf7c11c7, /127.1.193.126:50918 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.304 [DEBUG - New I/O worker #225] (Negotiator.java:764) Authenticated connection [id: 0x1efcaea1, /127.1.193.126:50915 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.036 [DEBUG - New I/O worker #407] (Negotiator.java:764) Authenticated connection [id: 0xb947a213, /127.1.193.125:50228 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.304 [DEBUG - New I/O worker #131] (Negotiator.java:764) Authenticated connection [id: 0x1c1bf8cf, /127.1.193.125:50125 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.303 [DEBUG - New I/O worker #199] (Negotiator.java:764) Authenticated connection [id: 0xe69ee55e, /127.1.193.126:50903 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.303 [DEBUG - New I/O worker #147] (Negotiator.java:764) Authenticated connection [id: 0x50bc57ba, /127.1.193.126:50894 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.302 [DEBUG - New I/O worker #255] (Negotiator.java:764) Authenticated connection [id: 0x8653ca42, /127.1.193.126:50897 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.302 [DEBUG - New I/O worker #102] (Negotiator.java:764) Authenticated connection [id: 0x33b2220e, /127.1.193.125:50188 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.301 [DEBUG - New I/O worker #273] (Negotiator.java:764) Authenticated connection [id: 0xf2fb6694, /127.1.193.125:50167 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.301 [DEBUG - New I/O worker #256] (Negotiator.java:764) Authenticated connection [id: 0xefabe5c6, /127.1.193.124:46125 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.300 [DEBUG - New I/O worker #238] (Negotiator.java:764) Authenticated connection [id: 0xd8b5986e, /127.1.193.125:50170 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.300 [DEBUG - New I/O worker #109] (Negotiator.java:764) Authenticated connection [id: 0xc3270375, /127.1.193.126:50879 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.300 [DEBUG - New I/O worker #399] (Negotiator.java:764) Authenticated connection [id: 0x4913a376, /127.1.193.126:50933 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.300 [DEBUG - New I/O worker #502] (Negotiator.java:764) Authenticated connection [id: 0xe0fb7861, /127.1.193.126:50936 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.299 [DEBUG - New I/O worker #181] (Negotiator.java:764) Authenticated connection [id: 0x1489d00e, /127.1.193.126:50900 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.299 [DEBUG - New I/O worker #191] (Negotiator.java:764) Authenticated connection [id: 0xace4b2b5, /127.1.193.126:50912 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.299 [DEBUG - New I/O worker #228] (Negotiator.java:764) Authenticated connection [id: 0x88376015, /127.1.193.126:50924 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.299 [DEBUG - New I/O worker #153] (Negotiator.java:764) Authenticated connection [id: 0xbd36556d, /127.1.193.124:46098 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.299 [DEBUG - New I/O worker #342] (Negotiator.java:764) Authenticated connection [id: 0xcfda2e77, /127.1.193.126:50945 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.299 [DEBUG - New I/O worker #192] (Negotiator.java:764) Authenticated connection [id: 0x87951176, /127.1.193.126:50927 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.296 [DEBUG - New I/O worker #130] (Negotiator.java:764) Authenticated connection [id: 0xf8d46163, /127.1.193.124:46107 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.296 [DEBUG - New I/O worker #203] (Negotiator.java:764) Authenticated connection [id: 0xfb57042f, /127.1.193.124:46071 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.295 [DEBUG - New I/O worker #157] (Negotiator.java:764) Authenticated connection [id: 0x39876e55, /127.1.193.126:50873 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.295 [DEBUG - New I/O worker #303] (Negotiator.java:764) Authenticated connection [id: 0xa3a26f20, /127.1.193.125:50206 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.295 [DEBUG - New I/O worker #137] (Negotiator.java:764) Authenticated connection [id: 0x3a819e48, /127.1.193.125:50191 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.047 [DEBUG - New I/O worker #424] (Negotiator.java:764) Authenticated connection [id: 0xb8cc3f45, /127.1.193.124:46153 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.293 [DEBUG - New I/O worker #184] (Negotiator.java:764) Authenticated connection [id: 0x27bc5e18, /127.1.193.124:46101 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.292 [DEBUG - New I/O worker #71] (Negotiator.java:764) Authenticated connection [id: 0xb2842ea4, /127.1.193.125:50143 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.292 [DEBUG - New I/O worker #105] (Negotiator.java:764) Authenticated connection [id: 0xf419fb72, /127.1.193.124:46104 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.291 [DEBUG - New I/O worker #398] (Negotiator.java:764) Authenticated connection [id: 0x23f685e5, /127.1.193.124:46119 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.290 [DEBUG - New I/O worker #480] (Negotiator.java:764) Authenticated connection [id: 0xabf9f478, /127.1.193.124:46122 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.255 [DEBUG - pool-3-thread-47] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.253 [DEBUG - New I/O worker #154] (Negotiator.java:764) Authenticated connection [id: 0x54eae8fe, /127.1.193.126:50861 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.054 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.054 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1930), Traces: [0ms] refreshing cache from master, [285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [853ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1931ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.055 [DEBUG - New I/O worker #102] (Deferred.java:1330) callback=retry RPC@1040441115 returned Deferred@977034243(state=PENDING, result=null, callback=(continuation of Deferred@2134824224 after retry RPC@1040441115), errback=(continuation of Deferred@2134824224 after retry RPC@1040441115)), so the following Deferred is getting paused: Deferred@2134824224(state=PAUSED, result=Deferred@977034243, callback=org.apache.kudu.client.AsyncKuduClient$6@6ed00a1b -> wakeup thread pool-3-thread-24, errback=passthrough -> wakeup thread pool-3-thread-24) 01:26:46.238 [DEBUG - New I/O worker #92] (Negotiator.java:764) Authenticated connection [id: 0x140bbca7, /127.1.193.126:50867 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.233 [DEBUG - New I/O worker #186] (Negotiator.java:764) Authenticated connection [id: 0x10161688, /127.1.193.124:46089 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.190 [DEBUG - New I/O worker #218] (Negotiator.java:764) Authenticated connection [id: 0x03086bd2, /127.1.193.124:46083 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.189 [DEBUG - New I/O worker #104] (Negotiator.java:764) Authenticated connection [id: 0x46aa37f3, /127.1.193.124:46080 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.183 [DEBUG - New I/O worker #78] (Negotiator.java:764) Authenticated connection [id: 0xc283cb25, /127.1.193.124:46068 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.183 [DEBUG - New I/O worker #222] (Negotiator.java:764) Authenticated connection [id: 0xd50afcb6, /127.1.193.126:50864 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.175 [DEBUG - New I/O worker #28] (Negotiator.java:764) Authenticated connection [id: 0x84e1ef6c, /127.1.193.124:46062 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.164 [DEBUG - New I/O worker #51] (Negotiator.java:764) Authenticated connection [id: 0x6b4b3287, /127.1.193.125:50140 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.148 [DEBUG - New I/O worker #136] (Negotiator.java:764) Authenticated connection [id: 0x1da1bb53, /127.1.193.125:50161 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.123 [DEBUG - New I/O worker #185] (Negotiator.java:764) Authenticated connection [id: 0x80502816, /127.1.193.124:46110 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.118 [DEBUG - New I/O worker #561] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.060 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.061 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1886), Traces: [0ms] refreshing cache from master, [231ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [231ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [231ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [1886ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.061 [DEBUG - New I/O worker #138] (Deferred.java:1330) callback=retry RPC@1181094966 returned Deferred@1113163810(state=PENDING, result=null, callback=(continuation of Deferred@123411981 after retry RPC@1181094966), errback=(continuation of Deferred@123411981 after retry RPC@1181094966)), so the following Deferred is getting paused: Deferred@123411981(state=PAUSED, result=Deferred@1113163810, callback=org.apache.kudu.client.AsyncKuduClient$6@40a7c6c7 -> wakeup thread pool-3-thread-31, errback=passthrough -> wakeup thread pool-3-thread-31) 01:26:46.118 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.062 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.062 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2367), Traces: [0ms] refreshing cache from master, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [16ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [611ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [2366ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.118 [DEBUG - New I/O worker #76] (Negotiator.java:764) Authenticated connection [id: 0xada9835f, /127.1.193.126:50858 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.063 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.063 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1763), Traces: [0ms] refreshing cache from master, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [671ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1762ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.063 [DEBUG - New I/O worker #273] (Deferred.java:1330) callback=retry RPC@2033326770 returned Deferred@2098041164(state=PENDING, result=null, callback=(continuation of Deferred@940511369 after retry RPC@2033326770), errback=(continuation of Deferred@940511369 after retry RPC@2033326770)), so the following Deferred is getting paused: Deferred@940511369(state=PAUSED, result=Deferred@2098041164, callback=org.apache.kudu.client.AsyncKuduClient$6@607329ae -> wakeup thread pool-3-thread-43, errback=passthrough -> wakeup thread pool-3-thread-43) 01:26:46.115 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.064 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1887), Traces: [0ms] refreshing cache from master, [213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [924ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1863ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.065 [DEBUG - New I/O worker #134] (Deferred.java:1330) callback=retry RPC@911019300 returned Deferred@846368565(state=PENDING, result=null, callback=(continuation of Deferred@2003830559 after retry RPC@911019300), errback=(continuation of Deferred@2003830559 after retry RPC@911019300)), so the following Deferred is getting paused: Deferred@2003830559(state=PAUSED, result=Deferred@846368565, callback=org.apache.kudu.client.AsyncKuduClient$6@18376d78 -> wakeup thread pool-3-thread-34, errback=passthrough -> wakeup thread pool-3-thread-34) 01:26:46.115 [DEBUG - New I/O worker #257] (Negotiator.java:764) Authenticated connection [id: 0x70ff07ab, /127.1.193.126:50870 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.113 [DEBUG - New I/O worker #347] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.066 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.067 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1691), Traces: [0ms] refreshing cache from master, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1692ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.067 [DEBUG - New I/O worker #238] (Deferred.java:1330) callback=retry RPC@608491611 returned Deferred@544950283(state=PENDING, result=null, callback=(continuation of Deferred@1702482528 after retry RPC@608491611), errback=(continuation of Deferred@1702482528 after retry RPC@608491611)), so the following Deferred is getting paused: Deferred@1702482528(state=PAUSED, result=Deferred@544950283, callback=org.apache.kudu.client.AsyncKuduClient$6@56d2ad4f -> wakeup thread pool-3-thread-42, errback=passthrough -> wakeup thread pool-3-thread-42) 01:26:46.113 [DEBUG - New I/O worker #510] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.068 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.069 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2384), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [549ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [2385ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.069 [DEBUG - New I/O worker #175] (Deferred.java:1330) callback=retry RPC@964874138 returned Deferred@1035820146(state=PENDING, result=null, callback=(continuation of Deferred@2025833889 after retry RPC@964874138), errback=(continuation of Deferred@2025833889 after retry RPC@964874138)), so the following Deferred is getting paused: Deferred@2025833889(state=PAUSED, result=Deferred@1035820146, callback=org.apache.kudu.client.AsyncKuduClient$6@665c0093 -> wakeup thread pool-3-thread-38, errback=passthrough -> wakeup thread pool-3-thread-38) 01:26:46.112 [DEBUG - New I/O worker #437] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.111 [DEBUG - New I/O worker #387] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:46.109 [DEBUG - New I/O worker #442] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.107 [DEBUG - pool-3-thread-70] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.107 [DEBUG - pool-3-thread-51] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.105 [DEBUG - pool-3-thread-52] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:46.105 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.073 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1422), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [451ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1395ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.073 [DEBUG - New I/O worker #258] (Deferred.java:1330) callback=retry RPC@798443760 returned Deferred@732482010(state=PENDING, result=null, callback=(continuation of Deferred@1856651979 after retry RPC@798443760), errback=(continuation of Deferred@1856651979 after retry RPC@798443760)), so the following Deferred is getting paused: Deferred@1856651979(state=PAUSED, result=Deferred@732482010, callback=org.apache.kudu.client.AsyncKuduClient$6@4acbf846 -> wakeup thread pool-3-thread-44, errback=passthrough -> wakeup thread pool-3-thread-44) 01:26:46.105 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.074 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1895), Traces: [0ms] refreshing cache from master, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [210ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [210ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [777ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1870ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.074 [DEBUG - New I/O worker #145] (Deferred.java:1330) callback=retry RPC@1282459967 returned Deferred@1213154409(state=PENDING, result=null, callback=(continuation of Deferred@223200004 after retry RPC@1282459967), errback=(continuation of Deferred@223200004 after retry RPC@1282459967)), so the following Deferred is getting paused: Deferred@223200004(state=PAUSED, result=Deferred@1213154409, callback=org.apache.kudu.client.AsyncKuduClient$6@76f9ea53 -> wakeup thread pool-3-thread-28, errback=passthrough -> wakeup thread pool-3-thread-28) 01:26:46.102 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.075 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1419), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [446ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1393ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.076 [DEBUG - New I/O worker #216] (Deferred.java:1330) callback=retry RPC@1037572946 returned Deferred@971474993(state=PENDING, result=null, callback=(continuation of Deferred@2095386985 after retry RPC@1037572946), errback=(continuation of Deferred@2095386985 after retry RPC@1037572946)), so the following Deferred is getting paused: Deferred@2095386985(state=PAUSED, result=Deferred@971474993, callback=org.apache.kudu.client.AsyncKuduClient$6@302ce05f -> wakeup thread pool-3-thread-40, errback=passthrough -> wakeup thread pool-3-thread-40) 01:26:46.099 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.076 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1413), Traces: [0ms] refreshing cache from master, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [13ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [13ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [436ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1393ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.077 [DEBUG - New I/O worker #394] (Deferred.java:1330) callback=retry RPC@464170733 returned Deferred@529869120(state=PENDING, result=null, callback=(continuation of Deferred@1519887574 after retry RPC@464170733), errback=(continuation of Deferred@1519887574 after retry RPC@464170733)), so the following Deferred is getting paused: Deferred@1519887574(state=PAUSED, result=Deferred@529869120, callback=org.apache.kudu.client.AsyncKuduClient$6@25141176 -> wakeup thread pool-3-thread-68, errback=passthrough -> wakeup thread pool-3-thread-68) 01:26:47.073 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.063 [DEBUG - New I/O worker #51] (Deferred.java:1330) callback=retry RPC@624388641 returned Deferred@554232415(state=PENDING, result=null, callback=(continuation of Deferred@1678401562 after retry RPC@624388641), errback=(continuation of Deferred@1678401562 after retry RPC@624388641)), so the following Deferred is getting paused: Deferred@1678401562(state=PAUSED, result=Deferred@554232415, callback=org.apache.kudu.client.AsyncKuduClient$6@4bdc6c4d -> wakeup thread pool-3-thread-19, errback=passthrough -> wakeup thread pool-3-thread-19) 01:26:47.062 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1885), Traces: [0ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [942ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1877ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.086 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.086 [DEBUG - New I/O worker #387] (Negotiator.java:764) Authenticated connection [id: 0x5bb448d2, /127.1.193.124:46137 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.096 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.046 [DEBUG - New I/O worker #492] (Negotiator.java:764) Authenticated connection [id: 0x620d3678, /127.1.193.126:50970 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.038 [DEBUG - New I/O worker #417] (Negotiator.java:764) Authenticated connection [id: 0x56a4a20c, /127.1.193.124:46156 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.100 [DEBUG - New I/O worker #561] (Negotiator.java:764) Authenticated connection [id: 0x934d68be, /127.1.193.126:50954 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.027 [DEBUG - New I/O worker #490] (Negotiator.java:764) Authenticated connection [id: 0x874daab2, /127.1.193.124:46144 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.025 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.101 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1186), Traces: [0ms] refreshing cache from master, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [1106ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1109ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1110ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.101 [DEBUG - New I/O worker #383] (Deferred.java:1330) callback=retry RPC@1989987626 returned Deferred@1923307297(state=PENDING, result=null, callback=(continuation of Deferred@933349137 after retry RPC@1989987626), errback=(continuation of Deferred@933349137 after retry RPC@1989987626)), so the following Deferred is getting paused: Deferred@933349137(state=PAUSED, result=Deferred@1923307297, callback=org.apache.kudu.client.AsyncKuduClient$6@16b9dc00 -> wakeup thread pool-3-thread-74, errback=passthrough -> wakeup thread pool-3-thread-74) 01:26:47.018 [DEBUG - New I/O worker #485] (Negotiator.java:764) Authenticated connection [id: 0x876bc2b6, /127.1.193.126:50967 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.006 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.103 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.103 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.103 [DEBUG - New I/O worker #26] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.104 [DEBUG - New I/O worker #510] (Negotiator.java:764) Authenticated connection [id: 0x8ba4b2d7, /127.1.193.124:46140 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.004 [DEBUG - New I/O worker #403] (Negotiator.java:764) Authenticated connection [id: 0x2fd3ac5e, /127.1.193.125:50225 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.003 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:46.996 [DEBUG - New I/O worker #537] (Negotiator.java:764) Authenticated connection [id: 0x53d682c3, /127.1.193.126:50961 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.107 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.108 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.108 [DEBUG - New I/O worker #54] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.108 [DEBUG - New I/O worker #54] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:46.996 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.110 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.110 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.110 [DEBUG - New I/O worker #19] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.111 [DEBUG - New I/O worker #19] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:46.992 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.112 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:46.989 [DEBUG - New I/O worker #526] (Negotiator.java:764) Authenticated connection [id: 0x0750b326, /127.1.193.126:50973 => /127.1.193.126:40739] using SASL/PLAIN 01:26:46.988 [DEBUG - New I/O worker #436] (Negotiator.java:764) Authenticated connection [id: 0x07f4ef1d, /127.1.193.125:50234 => /127.1.193.125:37435] using SASL/PLAIN 01:26:46.988 [DEBUG - New I/O worker #493] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:46.986 [DEBUG - New I/O worker #509] (Negotiator.java:764) Authenticated connection [id: 0x3237180a, /127.1.193.124:46159 => /127.1.193.124:58247] using SASL/PLAIN 01:26:46.967 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.939 [DEBUG - New I/O worker #10] (Deferred.java:1330) callback=retry RPC@303952470 returned Deferred@1461677681(state=PENDING, result=null, callback=(continuation of Deferred@1661464861 after retry RPC@303952470), errback=(continuation of Deferred@1661464861 after retry RPC@303952470)), so the following Deferred is getting paused: Deferred@1661464861(state=PAUSED, result=Deferred@1461677681, callback=org.apache.kudu.client.AsyncKuduClient$6@6682d903 -> wakeup thread pool-3-thread-1, errback=passthrough -> wakeup thread pool-3-thread-1) 01:26:47.114 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.937 [DEBUG - New I/O worker #353] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:46.927 [DEBUG - New I/O worker #55] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:46.923 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.923 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.913 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.117 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.117 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.117 [DEBUG - New I/O worker #98] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.118 [DEBUG - New I/O worker #98] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.118 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.913 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.119 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.119 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.120 [DEBUG - New I/O worker #16] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.120 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.912 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.120 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.121 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.121 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:46.912 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.122 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1571), Traces: [0ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [578ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1571ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:46.911 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.122 [DEBUG - New I/O worker #303] (Deferred.java:1330) callback=retry RPC@2089634958 returned Deferred@2024986567(state=PENDING, result=null, callback=(continuation of Deferred@1034962613 after retry RPC@2089634958), errback=(continuation of Deferred@1034962613 after retry RPC@2089634958)), so the following Deferred is getting paused: Deferred@1034962613(state=PAUSED, result=Deferred@2024986567, callback=org.apache.kudu.client.AsyncKuduClient$6@1a38fab0 -> wakeup thread pool-3-thread-45, errback=passthrough -> wakeup thread pool-3-thread-45) 01:26:47.123 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.123 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.124 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2403), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [536ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [2362ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [2403ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.124 [DEBUG - New I/O worker #71] (Deferred.java:1330) callback=retry RPC@1156268026 returned Deferred@1087677375(state=PENDING, result=null, callback=(continuation of Deferred@97924545 after retry RPC@1156268026), errback=(continuation of Deferred@97924545 after retry RPC@1156268026)), so the following Deferred is getting paused: Deferred@97924545(state=PAUSED, result=Deferred@1087677375, callback=org.apache.kudu.client.AsyncKuduClient$6@2990ff04 -> wakeup thread pool-3-thread-21, errback=passthrough -> wakeup thread pool-3-thread-21) 01:26:46.903 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.899 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.899 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.894 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.889 [DEBUG - New I/O worker #101] (Deferred.java:1330) callback=retry RPC@772785295 returned Deferred@2001383748(state=PENDING, result=null, callback=(continuation of Deferred@1458175404 after retry RPC@772785295), errback=(continuation of Deferred@1458175404 after retry RPC@772785295)), so the following Deferred is getting paused: Deferred@1458175404(state=PAUSED, result=Deferred@2001383748, callback=org.apache.kudu.client.AsyncKuduClient$6@26692992 -> wakeup thread pool-3-thread-25, errback=passthrough -> wakeup thread pool-3-thread-25) 01:26:46.889 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.126 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:46.886 [DEBUG - New I/O worker #13] (Deferred.java:1330) callback=retry RPC@1643490107 returned Deferred@1707750365(state=PENDING, result=null, callback=(continuation of Deferred@550019328 after retry RPC@1643490107), errback=(continuation of Deferred@550019328 after retry RPC@1643490107)), so the following Deferred is getting paused: Deferred@550019328(state=PAUSED, result=Deferred@1707750365, callback=org.apache.kudu.client.AsyncKuduClient$6@5dc27284 -> wakeup thread pool-3-thread-4, errback=passthrough -> wakeup thread pool-3-thread-4) 01:26:47.127 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.128 [DEBUG - New I/O worker #413] (Negotiator.java:764) Authenticated connection [id: 0xa60087fb, /127.1.193.125:50231 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.127 [DEBUG - New I/O worker #353] (Negotiator.java:764) Authenticated connection [id: 0x356939af, /127.1.193.125:50237 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.126 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.123 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.122 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.131 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2498), Traces: [0ms] refreshing cache from master, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [635ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [2491ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [2491ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.122 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.131 [DEBUG - New I/O worker #131] (Deferred.java:1330) callback=retry RPC@84064335 returned Deferred@20790327(state=PENDING, result=null, callback=(continuation of Deferred@1145025140 after retry RPC@84064335), errback=(continuation of Deferred@1145025140 after retry RPC@84064335)), so the following Deferred is getting paused: Deferred@1145025140(state=PAUSED, result=Deferred@20790327, callback=org.apache.kudu.client.AsyncKuduClient$6@65e1aadb -> wakeup thread pool-3-thread-35, errback=passthrough -> wakeup thread pool-3-thread-35) 01:26:47.121 [DEBUG - New I/O worker #75] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.132 [DEBUG - New I/O worker #75] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.121 [DEBUG - New I/O worker #437] (Negotiator.java:764) Authenticated connection [id: 0xb159a07b, /127.1.193.125:50215 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.120 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2001), Traces: [0ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [2001ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.120 [DEBUG - New I/O worker #16] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.117 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.135 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.135 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.114 [DEBUG - New I/O worker #442] (Negotiator.java:764) Authenticated connection [id: 0x0ec34b77, /127.1.193.126:50951 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.112 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.137 [DEBUG - New I/O worker #46] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.137 [DEBUG - New I/O worker #46] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.104 [DEBUG - New I/O worker #26] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.086 [DEBUG - New I/O worker #49] (Deferred.java:1330) callback=retry RPC@1411162142 returned Deferred@1344482885(state=PENDING, result=null, callback=(continuation of Deferred@354527781 after retry RPC@1411162142), errback=(continuation of Deferred@354527781 after retry RPC@1411162142)), so the following Deferred is getting paused: Deferred@354527781(state=PAUSED, result=Deferred@1344482885, callback=org.apache.kudu.client.AsyncKuduClient$6@15d3b759 -> wakeup thread pool-3-thread-20, errback=passthrough -> wakeup thread pool-3-thread-20) 01:26:47.136 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.135 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.133 [DEBUG - New I/O worker #137] (Deferred.java:1330) callback=retry RPC@148340006 returned Deferred@216590564(state=PENDING, result=null, callback=(continuation of Deferred@1240102685 after retry RPC@148340006), errback=(continuation of Deferred@1240102685 after retry RPC@148340006)), so the following Deferred is getting paused: Deferred@1240102685(state=PAUSED, result=Deferred@216590564, callback=org.apache.kudu.client.AsyncKuduClient$6@4b644197 -> wakeup thread pool-3-thread-32, errback=passthrough -> wakeup thread pool-3-thread-32) 01:26:47.133 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.144 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1443), Traces: [0ms] refreshing cache from master, [405ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1432ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1436ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.144 [DEBUG - New I/O worker #437] (Deferred.java:1330) callback=retry RPC@1300310768 returned Deferred@1237231512(state=PENDING, result=null, callback=(continuation of Deferred@213655755 after retry RPC@1300310768), errback=(continuation of Deferred@213655755 after retry RPC@1300310768)), so the following Deferred is getting paused: Deferred@213655755(state=PAUSED, result=Deferred@1237231512, callback=org.apache.kudu.client.AsyncKuduClient$6@79688c38 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:26:47.147 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.153 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.154 [DEBUG - New I/O worker #420] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.154 [DEBUG - New I/O worker #491] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.132 [DEBUG - New I/O worker #542] (Negotiator.java:764) Authenticated connection [id: 0xac054ad1, /127.1.193.126:50958 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.131 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.157 [DEBUG - New I/O worker #73] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.157 [DEBUG - New I/O worker #73] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.131 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.159 [DEBUG - New I/O worker #56] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.131 [DEBUG - New I/O worker #347] (Negotiator.java:764) Authenticated connection [id: 0x8f648d78, /127.1.193.125:50212 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.130 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.130 [DEBUG - New I/O worker #23] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.160 [DEBUG - New I/O worker #23] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.159 [DEBUG - New I/O worker #56] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.156 [DEBUG - New I/O worker #427] (Negotiator.java:764) Authenticated connection [id: 0xbf7ab28c, /127.1.193.124:46150 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.162 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.162 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.163 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.163 [DEBUG - New I/O worker #493] (Negotiator.java:764) Authenticated connection [id: 0x3162584c, /127.1.193.126:50976 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.165 [DEBUG - New I/O worker #352] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.165 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.166 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=2329), Traces: [0ms] refreshing cache from master, [133ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [133ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [399ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [2256ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [2328ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.166 [DEBUG - New I/O worker #136] (Deferred.java:1330) callback=retry RPC@847958222 returned Deferred@917848647(state=PENDING, result=null, callback=(continuation of Deferred@1941428981 after retry RPC@847958222), errback=(continuation of Deferred@1941428981 after retry RPC@847958222)), so the following Deferred is getting paused: Deferred@1941428981(state=PAUSED, result=Deferred@917848647, callback=org.apache.kudu.client.AsyncKuduClient$6@26e65125 -> wakeup thread pool-3-thread-33, errback=passthrough -> wakeup thread pool-3-thread-33) 01:26:47.167 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.173 [DEBUG - New I/O worker #491] (Negotiator.java:764) Authenticated connection [id: 0x0c797b5d, /127.1.193.126:50979 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.178 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.180 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.180 [DEBUG - New I/O worker #420] (Negotiator.java:764) Authenticated connection [id: 0x7a566978, /127.1.193.124:46165 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.182 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.182 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.183 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1465), Traces: [0ms] refreshing cache from master, [389ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [390ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [390ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1428ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.183 [DEBUG - New I/O worker #347] (Deferred.java:1330) callback=retry RPC@694828438 returned Deferred@760593507(state=PENDING, result=null, callback=(continuation of Deferred@1750546349 after retry RPC@694828438), errback=(continuation of Deferred@1750546349 after retry RPC@694828438)), so the following Deferred is getting paused: Deferred@1750546349(state=PAUSED, result=Deferred@760593507, callback=org.apache.kudu.client.AsyncKuduClient$6@218b60fb -> wakeup thread pool-3-thread-50, errback=passthrough -> wakeup thread pool-3-thread-50) 01:26:47.186 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.189 [DEBUG - New I/O worker #352] (Negotiator.java:764) Authenticated connection [id: 0x863986d1, /127.1.193.125:50240 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.198 [DEBUG - New I/O worker #350] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.201 [DEBUG - New I/O worker #432] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.222 [DEBUG - New I/O worker #395] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.223 [DEBUG - New I/O worker #432] (Negotiator.java:764) Authenticated connection [id: 0xc9a2a07f, /127.1.193.126:50982 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.222 [DEBUG - New I/O worker #350] (Negotiator.java:764) Authenticated connection [id: 0x6536c035, /127.1.193.125:50243 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.248 [DEBUG - New I/O worker #440] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.249 [DEBUG - New I/O worker #361] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.249 [DEBUG - New I/O worker #393] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.248 [DEBUG - New I/O worker #308] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.248 [DEBUG - New I/O worker #380] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.456 [DEBUG - New I/O worker #395] (Negotiator.java:764) Authenticated connection [id: 0xad424600, /127.1.193.124:46168 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.447 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.445 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.461 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.461 [DEBUG - pool-3-thread-7] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.462 [DEBUG - New I/O worker #80] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.463 [DEBUG - New I/O worker #34] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.289 [DEBUG - New I/O worker #54] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.444 [DEBUG - pool-3-thread-39] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.443 [DEBUG - New I/O worker #203] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.379 [DEBUG - New I/O worker #481] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.466 [DEBUG - New I/O worker #357] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.468 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.468 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.468 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.469 [DEBUG - New I/O worker #273] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.469 [DEBUG - New I/O worker #273] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.469 [DEBUG - New I/O worker #141] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.373 [DEBUG - New I/O worker #354] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.372 [DEBUG - New I/O worker #478] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.473 [DEBUG - pool-3-thread-22] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.473 [DEBUG - New I/O worker #110] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.474 [DEBUG - pool-3-thread-10] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.474 [DEBUG - New I/O worker #29] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.294 [DEBUG - New I/O worker #16] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.361 [DEBUG - New I/O worker #349] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.476 [DEBUG - pool-3-thread-14] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.476 [DEBUG - pool-3-thread-9] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.294 [DEBUG - New I/O worker #46] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.477 [DEBUG - New I/O worker #40] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.295 [DEBUG - New I/O worker #26] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.355 [DEBUG - New I/O worker #441] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.478 [DEBUG - pool-3-thread-23] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.479 [DEBUG - New I/O worker #104] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.479 [DEBUG - pool-3-thread-12] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.479 [DEBUG - New I/O worker #30] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.479 [DEBUG - pool-3-thread-17] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.297 [DEBUG - New I/O worker #23] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.480 [DEBUG - New I/O worker #82] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.297 [DEBUG - New I/O worker #56] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.297 [DEBUG - New I/O worker #73] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.354 [DEBUG - New I/O worker #423] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.332 [DEBUG - New I/O worker #474] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.331 [DEBUG - New I/O worker #506] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.330 [DEBUG - New I/O worker #433] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.330 [DEBUG - New I/O worker #569] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.488 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.488 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.489 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.489 [DEBUG - New I/O worker #394] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.490 [DEBUG - New I/O worker #394] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.324 [DEBUG - New I/O worker #260] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.491 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.492 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.492 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.492 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.493 [DEBUG - New I/O worker #136] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.493 [DEBUG - New I/O worker #136] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.316 [DEBUG - New I/O worker #386] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.314 [DEBUG - New I/O worker #239] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.495 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.496 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.496 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.496 [DEBUG - New I/O worker #383] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.497 [DEBUG - New I/O worker #383] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.314 [DEBUG - New I/O worker #305] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.312 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.500 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.500 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.500 [DEBUG - New I/O worker #15] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.501 [DEBUG - New I/O worker #15] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.311 [DEBUG - New I/O worker #503] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.311 [DEBUG - New I/O worker #355] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.310 [DEBUG - New I/O worker #430] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.309 [DEBUG - New I/O worker #356] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.309 [DEBUG - New I/O worker #431] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.308 [DEBUG - New I/O worker #397] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.307 [DEBUG - New I/O worker #412] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.307 [DEBUG - New I/O worker #415] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.307 [DEBUG - New I/O worker #409] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.306 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.513 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.513 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.513 [DEBUG - New I/O worker #21] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.514 [DEBUG - New I/O worker #21] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.304 [DEBUG - New I/O worker #426] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.303 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.515 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.515 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.516 [DEBUG - New I/O worker #106] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.516 [DEBUG - New I/O worker #106] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.302 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.517 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.517 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.518 [DEBUG - New I/O worker #176] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.518 [DEBUG - New I/O worker #176] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.301 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.519 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.519 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.520 [DEBUG - New I/O worker #27] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.520 [DEBUG - New I/O worker #27] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.301 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.521 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=448), Traces: [0ms] refreshing cache from master, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [161ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [411ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.522 [DEBUG - New I/O worker #350] (Deferred.java:1330) callback=retry RPC@2040112382 returned Deferred@2108043029(state=PENDING, result=null, callback=(continuation of Deferred@950316741 after retry RPC@2040112382), errback=(continuation of Deferred@950316741 after retry RPC@2040112382)), so the following Deferred is getting paused: Deferred@950316741(state=PAUSED, result=Deferred@2108043029, callback=org.apache.kudu.client.AsyncKuduClient$6@3ddcb8d8 -> wakeup thread pool-3-thread-52, errback=passthrough -> wakeup thread pool-3-thread-52) 01:26:47.300 [DEBUG - New I/O worker #422] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.300 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.525 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=513), Traces: [0ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [165ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [218ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.525 [DEBUG - New I/O worker #352] (Deferred.java:1330) callback=retry RPC@1201672296 returned Deferred@1134533700(state=PENDING, result=null, callback=(continuation of Deferred@110959187 after retry RPC@1201672296), errback=(continuation of Deferred@110959187 after retry RPC@1201672296)), so the following Deferred is getting paused: Deferred@110959187(state=PAUSED, result=Deferred@1134533700, callback=org.apache.kudu.client.AsyncKuduClient$6@70b593e3 -> wakeup thread pool-3-thread-55, errback=passthrough -> wakeup thread pool-3-thread-55) 01:26:47.300 [DEBUG - New I/O worker #487] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.300 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.299 [DEBUG - New I/O worker #404] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.528 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=604), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [124ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [247ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [253ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.529 [DEBUG - New I/O worker #353] (Deferred.java:1330) callback=retry RPC@437173583 returned Deferred@506538710(state=PENDING, result=null, callback=(continuation of Deferred@1530115956 after retry RPC@437173583), errback=(continuation of Deferred@1530115956 after retry RPC@437173583)), so the following Deferred is getting paused: Deferred@1530115956(state=PAUSED, result=Deferred@506538710, callback=org.apache.kudu.client.AsyncKuduClient$6@5a919c5 -> wakeup thread pool-3-thread-46, errback=passthrough -> wakeup thread pool-3-thread-46) 01:26:47.299 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.531 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=828), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [440ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [467ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [469ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.531 [DEBUG - New I/O worker #436] (Deferred.java:1330) callback=retry RPC@1416383859 returned Deferred@1347670822(state=PENDING, result=null, callback=(continuation of Deferred@357648200 after retry RPC@1416383859), errback=(continuation of Deferred@357648200 after retry RPC@1416383859)), so the following Deferred is getting paused: Deferred@357648200(state=PAUSED, result=Deferred@1347670822, callback=org.apache.kudu.client.AsyncKuduClient$6@123f34bc -> wakeup thread pool-3-thread-76, errback=passthrough -> wakeup thread pool-3-thread-76) 01:26:47.299 [DEBUG - New I/O worker #428] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.298 [DEBUG - New I/O worker #416] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.296 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.535 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=832), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [432ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [452ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [457ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.535 [DEBUG - New I/O worker #413] (Deferred.java:1330) callback=retry RPC@577875047 returned Deferred@642661617(state=PENDING, result=null, callback=(continuation of Deferred@1665967708 after retry RPC@577875047), errback=(continuation of Deferred@1665967708 after retry RPC@577875047)), so the following Deferred is getting paused: Deferred@1665967708(state=PAUSED, result=Deferred@642661617, callback=org.apache.kudu.client.AsyncKuduClient$6@7dd8efeb -> wakeup thread pool-3-thread-58, errback=passthrough -> wakeup thread pool-3-thread-58) 01:26:47.296 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.536 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=831), Traces: [0ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [436ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [443ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [452ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.537 [DEBUG - New I/O worker #407] (Deferred.java:1330) callback=retry RPC@127016440 returned Deferred@61708754(state=PENDING, result=null, callback=(continuation of Deferred@1185880003 after retry RPC@127016440), errback=(continuation of Deferred@1185880003 after retry RPC@127016440)), so the following Deferred is getting paused: Deferred@1185880003(state=PAUSED, result=Deferred@61708754, callback=org.apache.kudu.client.AsyncKuduClient$6@60493d56 -> wakeup thread pool-3-thread-60, errback=passthrough -> wakeup thread pool-3-thread-60) 01:26:47.295 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.292 [DEBUG - New I/O worker #541] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.537 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=834), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [332ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [439ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [463ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.538 [DEBUG - New I/O worker #403] (Deferred.java:1330) callback=retry RPC@637236424 returned Deferred@566551727(state=PENDING, result=null, callback=(continuation of Deferred@1690730227 after retry RPC@637236424), errback=(continuation of Deferred@1690730227 after retry RPC@637236424)), so the following Deferred is getting paused: Deferred@1690730227(state=PAUSED, result=Deferred@566551727, callback=org.apache.kudu.client.AsyncKuduClient$6@2b4b57c3 -> wakeup thread pool-3-thread-62, errback=passthrough -> wakeup thread pool-3-thread-62) 01:26:47.290 [DEBUG - New I/O worker #489] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.290 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.289 [DEBUG - New I/O worker #414] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.288 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.543 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=839), Traces: [0ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [313ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [319ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [437ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.544 [DEBUG - New I/O worker #400] (Deferred.java:1330) callback=retry RPC@668728847 returned Deferred@602172972(state=PENDING, result=null, callback=(continuation of Deferred@1726411828 after retry RPC@668728847), errback=(continuation of Deferred@1726411828 after retry RPC@668728847)), so the following Deferred is getting paused: Deferred@1726411828(state=PAUSED, result=Deferred@602172972, callback=org.apache.kudu.client.AsyncKuduClient$6@2f193b51 -> wakeup thread pool-3-thread-63, errback=passthrough -> wakeup thread pool-3-thread-63) 01:26:47.288 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.544 [DEBUG - New I/O worker #307] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.288 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.546 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.547 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.547 [DEBUG - New I/O worker #18] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.548 [DEBUG - New I/O worker #18] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.285 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.285 [DEBUG - New I/O worker #174] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.249 [DEBUG - New I/O worker #434] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.548 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=847), Traces: [0ms] refreshing cache from master, [11ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [12ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [327ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [447ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [459ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.551 [DEBUG - New I/O worker #411] (Deferred.java:1330) callback=retry RPC@798682739 returned Deferred@732266328(state=PENDING, result=null, callback=(continuation of Deferred@1856496712 after retry RPC@798682739), errback=(continuation of Deferred@1856496712 after retry RPC@798682739)), so the following Deferred is getting paused: Deferred@1856496712(state=PAUSED, result=Deferred@732266328, callback=org.apache.kudu.client.AsyncKuduClient$6@279251e0 -> wakeup thread pool-3-thread-65, errback=passthrough -> wakeup thread pool-3-thread-65) 01:26:47.544 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.552 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.552 [DEBUG - New I/O worker #37] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.553 [DEBUG - New I/O worker #37] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.541 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.554 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.554 [DEBUG - New I/O worker #12] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.555 [DEBUG - New I/O worker #12] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.570 [DEBUG - New I/O worker #428] (Negotiator.java:764) Authenticated connection [id: 0x391cad12, /127.1.193.126:50997 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.574 [DEBUG - New I/O worker #440] (Negotiator.java:764) Authenticated connection [id: 0x9b771098, /127.1.193.126:50988 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.575 [DEBUG - New I/O worker #445] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.576 [DEBUG - New I/O worker #409] (Negotiator.java:764) Authenticated connection [id: 0x81f29531, /127.1.193.125:50261 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.577 [DEBUG - New I/O worker #434] (Negotiator.java:764) Authenticated connection [id: 0x2053aa72, /127.1.193.124:46173 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.581 [DEBUG - New I/O worker #441] (Negotiator.java:764) Authenticated connection [id: 0xa83a4285, /127.1.193.126:51018 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.582 [DEBUG - New I/O worker #354] (Negotiator.java:764) Authenticated connection [id: 0x502a331c, /127.1.193.125:50282 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.583 [DEBUG - New I/O worker #393] (Negotiator.java:764) Authenticated connection [id: 0x496663a9, /127.1.193.125:50247 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.583 [DEBUG - New I/O worker #481] (Negotiator.java:764) Authenticated connection [id: 0xd5d88c1f, /127.1.193.126:51024 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.584 [DEBUG - New I/O worker #397] (Negotiator.java:764) Authenticated connection [id: 0xfecb32c1, /127.1.193.124:46189 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.584 [DEBUG - New I/O worker #415] (Negotiator.java:764) Authenticated connection [id: 0x156cb88e, /127.1.193.126:51000 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.585 [DEBUG - New I/O worker #541] (Negotiator.java:764) Authenticated connection [id: 0x03bbe2fd, /127.1.193.126:50991 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.586 [DEBUG - New I/O worker #305] (Negotiator.java:764) Authenticated connection [id: 0x81908201, /127.1.193.126:51009 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.586 [DEBUG - New I/O worker #422] (Negotiator.java:764) Authenticated connection [id: 0xa4da0d65, /127.1.193.124:46180 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.528 [DEBUG - New I/O worker #239] (Negotiator.java:764) Authenticated connection [id: 0x70b35bf5, /127.1.193.125:50270 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.515 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.588 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.588 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.588 [DEBUG - New I/O worker #134] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.589 [DEBUG - New I/O worker #134] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.514 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.590 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.590 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.591 [DEBUG - New I/O worker #216] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.591 [DEBUG - New I/O worker #430] (Negotiator.java:764) Authenticated connection [id: 0x0df301ea, /127.1.193.124:46192 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.492 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.479 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.592 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.593 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.593 [DEBUG - New I/O worker #139] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.594 [DEBUG - New I/O worker #139] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.293 [DEBUG - New I/O worker #75] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.292 [DEBUG - New I/O worker #98] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.469 [DEBUG - pool-3-thread-26] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.292 [DEBUG - New I/O worker #55] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.465 [DEBUG - New I/O worker #72] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.465 [DEBUG - pool-3-thread-16] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.290 [DEBUG - New I/O worker #19] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.461 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.596 [DEBUG - New I/O worker #396] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.596 [DEBUG - New I/O worker #396] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.597 [DEBUG - New I/O worker #431] (Negotiator.java:764) Authenticated connection [id: 0x8754c332, /127.1.193.126:51003 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.461 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.461 [DEBUG - pool-3-thread-18] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.598 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.592 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.598 [DEBUG - New I/O worker #13] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.599 [DEBUG - New I/O worker #13] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.600 [DEBUG - New I/O worker #506] (Negotiator.java:764) Authenticated connection [id: 0x48e490fc, /127.1.193.126:51012 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.616 [DEBUG - New I/O worker #308] (Negotiator.java:764) Authenticated connection [id: 0x48c4afb4, /127.1.193.125:50246 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.618 [DEBUG - New I/O worker #416] (Negotiator.java:764) Authenticated connection [id: 0x6cb48a05, /127.1.193.125:50255 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.592 [DEBUG - New I/O worker #404] (Negotiator.java:764) Authenticated connection [id: 0xfd51595b, /127.1.193.125:50257 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.591 [DEBUG - New I/O worker #216] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.621 [DEBUG - New I/O worker #414] (Negotiator.java:764) Authenticated connection [id: 0xc77cb100, /127.1.193.125:50252 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.590 [DEBUG - New I/O worker #487] (Negotiator.java:764) Authenticated connection [id: 0x0bedc173, /127.1.193.126:50995 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.625 [DEBUG - New I/O worker #433] (Negotiator.java:764) Authenticated connection [id: 0x97bd50dd, /127.1.193.125:50276 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.626 [DEBUG - New I/O worker #445] (Negotiator.java:764) Authenticated connection [id: 0xed00b551, /127.1.193.126:51045 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.628 [DEBUG - New I/O worker #355] (Negotiator.java:764) Authenticated connection [id: 0x7fcb23c0, /127.1.193.125:50267 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.629 [DEBUG - New I/O worker #386] (Negotiator.java:764) Authenticated connection [id: 0x2a1eaa15, /127.1.193.125:50273 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.631 [DEBUG - New I/O worker #356] (Negotiator.java:764) Authenticated connection [id: 0xc514014e, /127.1.193.125:50264 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.631 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.632 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.632 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.632 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.633 [DEBUG - New I/O worker #51] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.633 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.633 [DEBUG - New I/O worker #51] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.634 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.635 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.635 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.636 [DEBUG - New I/O worker #357] (Negotiator.java:764) Authenticated connection [id: 0x5f9d257e, /127.1.193.124:46231 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.589 [DEBUG - New I/O worker #489] (Negotiator.java:764) Authenticated connection [id: 0xf7cfcb49, /127.1.193.124:46177 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.636 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.637 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.588 [DEBUG - New I/O worker #503] (Negotiator.java:764) Authenticated connection [id: 0x72b3038a, /127.1.193.126:51006 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.581 [DEBUG - New I/O worker #412] (Negotiator.java:764) Authenticated connection [id: 0x44671bf4, /127.1.193.124:46186 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.575 [DEBUG - New I/O worker #474] (Negotiator.java:764) Authenticated connection [id: 0x468b1b20, /127.1.193.124:46198 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.638 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.639 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.640 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.640 [DEBUG - New I/O worker #49] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.640 [DEBUG - New I/O worker #49] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.641 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.641 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.641 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.638 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.637 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.642 [DEBUG - New I/O worker #303] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.643 [DEBUG - New I/O worker #303] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.643 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.643 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.636 [DEBUG - New I/O worker #347] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.644 [DEBUG - New I/O worker #347] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.645 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.645 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.633 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.632 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.631 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.646 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.646 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.647 [DEBUG - New I/O worker #437] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.647 [DEBUG - New I/O worker #437] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.648 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.648 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.649 [DEBUG - New I/O worker #245] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.614 [DEBUG - New I/O worker #349] (Negotiator.java:764) Authenticated connection [id: 0x044b9e1f, /127.1.193.125:50283 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.611 [DEBUG - New I/O worker #260] (Negotiator.java:764) Authenticated connection [id: 0x5d63371a, /127.1.193.124:46195 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.651 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.651 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.652 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=621), Traces: [0ms] refreshing cache from master, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.652 [DEBUG - New I/O worker #307] (Negotiator.java:764) Authenticated connection [id: 0xdcde2916, /127.1.193.125:50306 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.652 [DEBUG - New I/O worker #404] (Deferred.java:1330) callback=retry RPC@1668538059 returned Deferred@1733052029(state=PENDING, result=null, callback=(continuation of Deferred@575591664 after retry RPC@1668538059), errback=(continuation of Deferred@575591664 after retry RPC@1668538059)), so the following Deferred is getting paused: Deferred@575591664(state=PAUSED, result=Deferred@1733052029, callback=org.apache.kudu.client.AsyncKuduClient$6@6e5e59f1 -> wakeup thread pool-3-thread-67, errback=passthrough -> wakeup thread pool-3-thread-67) 01:26:47.610 [DEBUG - New I/O worker #380] (Negotiator.java:764) Authenticated connection [id: 0xf0f72ee6, /127.1.193.126:50987 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.606 [DEBUG - New I/O worker #423] (Negotiator.java:764) Authenticated connection [id: 0xe9220ad0, /127.1.193.126:51023 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.605 [DEBUG - New I/O worker #361] (Negotiator.java:764) Authenticated connection [id: 0x0adab926, /127.1.193.124:46172 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.604 [DEBUG - New I/O worker #569] (Negotiator.java:764) Authenticated connection [id: 0x1c1e1f3b, /127.1.193.126:51015 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.602 [DEBUG - New I/O worker #426] (Negotiator.java:764) Authenticated connection [id: 0x7c41e4ed, /127.1.193.124:46183 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.657 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.657 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=632), Traces: [0ms] refreshing cache from master, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [577ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [632ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.601 [DEBUG - New I/O worker #478] (Negotiator.java:764) Authenticated connection [id: 0xb4a8943d, /127.1.193.124:46201 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.598 [DEBUG - New I/O worker #138] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.658 [DEBUG - New I/O worker #138] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.658 [DEBUG - New I/O worker #355] (Deferred.java:1330) callback=retry RPC@943364849 returned Deferred@1006967086(state=PENDING, result=null, callback=(continuation of Deferred@2030538954 after retry RPC@943364849), errback=(continuation of Deferred@2030538954 after retry RPC@943364849)), so the following Deferred is getting paused: Deferred@2030538954(state=PAUSED, result=Deferred@1006967086, callback=org.apache.kudu.client.AsyncKuduClient$6@24357236 -> wakeup thread pool-3-thread-59, errback=passthrough -> wakeup thread pool-3-thread-59) 01:26:47.651 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.651 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.660 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=628), Traces: [0ms] refreshing cache from master, [252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [571ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.660 [DEBUG - New I/O worker #414] (Deferred.java:1330) callback=retry RPC@581726114 returned Deferred@647233327(state=PENDING, result=null, callback=(continuation of Deferred@1670473113 after retry RPC@581726114), errback=(continuation of Deferred@1670473113 after retry RPC@581726114)), so the following Deferred is getting paused: Deferred@1670473113(state=PAUSED, result=Deferred@647233327, callback=org.apache.kudu.client.AsyncKuduClient$6@1328dcd8 -> wakeup thread pool-3-thread-57, errback=passthrough -> wakeup thread pool-3-thread-57) 01:26:47.649 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=628), Traces: [0ms] refreshing cache from master, [319ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.646 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.662 [DEBUG - New I/O worker #354] (Deferred.java:1330) callback=retry RPC@787582949 returned Deferred@718147299(state=PENDING, result=null, callback=(continuation of Deferred@1875674590 after retry RPC@787582949), errback=(continuation of Deferred@1875674590 after retry RPC@787582949)), so the following Deferred is getting paused: Deferred@1875674590(state=PAUSED, result=Deferred@718147299, callback=org.apache.kudu.client.AsyncKuduClient$6@7cc56f49 -> wakeup thread pool-3-thread-54, errback=passthrough -> wakeup thread pool-3-thread-54) 01:26:47.646 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.645 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.663 [DEBUG - New I/O worker #131] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.663 [DEBUG - New I/O worker #131] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.664 [DEBUG - New I/O worker #482] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.668 [DEBUG - New I/O worker #476] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.669 [DEBUG - New I/O worker #391] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.670 [DEBUG - New I/O worker #531] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.672 [DEBUG - New I/O worker #486] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.674 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.674 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=654), Traces: [0ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [582ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [654ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.675 [DEBUG - New I/O worker #349] (Deferred.java:1330) callback=retry RPC@33060186 returned Deferred@96989629(state=PENDING, result=null, callback=(continuation of Deferred@1086685025 after retry RPC@33060186), errback=(continuation of Deferred@1086685025 after retry RPC@33060186)), so the following Deferred is getting paused: Deferred@1086685025(state=PAUSED, result=Deferred@96989629, callback=org.apache.kudu.client.AsyncKuduClient$6@335f6c46 -> wakeup thread pool-3-thread-48, errback=passthrough -> wakeup thread pool-3-thread-48) 01:26:47.676 [DEBUG - New I/O worker #570] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.678 [DEBUG - New I/O worker #560] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.681 [DEBUG - New I/O worker #388] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.644 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.644 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.682 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=652), Traces: [0ms] refreshing cache from master, [274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [612ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [614ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.683 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.684 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.685 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.644 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.685 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=613), Traces: [0ms] refreshing cache from master, [170ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [170ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [590ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [612ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.644 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.686 [DEBUG - New I/O worker #308] (Deferred.java:1330) callback=retry RPC@1055546037 returned Deferred@987093424(state=PENDING, result=null, callback=(continuation of Deferred@2144817294 after retry RPC@1055546037), errback=(continuation of Deferred@2144817294 after retry RPC@1055546037)), so the following Deferred is getting paused: Deferred@2144817294(state=PAUSED, result=Deferred@987093424, callback=org.apache.kudu.client.AsyncKuduClient$6@22b15606 -> wakeup thread pool-3-thread-51, errback=passthrough -> wakeup thread pool-3-thread-51) 01:26:47.642 [DEBUG - New I/O worker #137] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.687 [DEBUG - New I/O worker #137] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.687 [DEBUG - New I/O worker #482] (Negotiator.java:764) Authenticated connection [id: 0x748aa40e, /127.1.193.124:46222 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.688 [DEBUG - New I/O worker #392] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.690 [DEBUG - New I/O worker #390] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.691 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.692 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=663), Traces: [0ms] refreshing cache from master, [276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [572ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [663ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.694 [DEBUG - New I/O worker #406] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.695 [DEBUG - New I/O worker #488] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.696 [DEBUG - New I/O worker #560] (Negotiator.java:764) Authenticated connection [id: 0x247adc27, /127.1.193.126:51033 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.697 [DEBUG - New I/O worker #475] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.698 [DEBUG - New I/O worker #438] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.698 [DEBUG - New I/O worker #539] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.699 [DEBUG - New I/O worker #390] (Negotiator.java:764) Authenticated connection [id: 0x43846a26, /127.1.193.125:50288 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.701 [DEBUG - New I/O worker #532] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.701 [DEBUG - New I/O worker #495] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.703 [DEBUG - New I/O worker #410] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.642 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.701 [DEBUG - New I/O worker #527] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.699 [DEBUG - New I/O worker #348] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.707 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=683), Traces: [0ms] refreshing cache from master, [300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [618ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [635ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:47.707 [DEBUG - New I/O worker #433] (Deferred.java:1330) callback=retry RPC@977112023 returned Deferred@1040320584(state=PENDING, result=null, callback=(continuation of Deferred@2063630828 after retry RPC@977112023), errback=(continuation of Deferred@2063630828 after retry RPC@977112023)), so the following Deferred is getting paused: Deferred@2063630828(state=PAUSED, result=Deferred@1040320584, callback=org.apache.kudu.client.AsyncKuduClient$6@1c84ce17 -> wakeup thread pool-3-thread-78, errback=passthrough -> wakeup thread pool-3-thread-78) 01:26:47.694 [DEBUG - New I/O worker #356] (Deferred.java:1330) callback=retry RPC@476097544 returned Deferred@408894300(state=PENDING, result=null, callback=(continuation of Deferred@1566418483 after retry RPC@476097544), errback=(continuation of Deferred@1566418483 after retry RPC@476097544)), so the following Deferred is getting paused: Deferred@1566418483(state=PAUSED, result=Deferred@408894300, callback=org.apache.kudu.client.AsyncKuduClient$6@7553ab84 -> wakeup thread pool-3-thread-53, errback=passthrough -> wakeup thread pool-3-thread-53) 01:26:47.688 [DEBUG - New I/O worker #389] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.686 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.685 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=655), Traces: [0ms] refreshing cache from master, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [571ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [614ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.853 [DEBUG - New I/O worker #532] (Negotiator.java:764) Authenticated connection [id: 0x8997a992, /127.1.193.126:51042 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.853 [DEBUG - New I/O worker #416] (Deferred.java:1330) callback=retry RPC@1344178849 returned Deferred@1411449978(state=PENDING, result=null, callback=(continuation of Deferred@287539354 after retry RPC@1344178849), errback=(continuation of Deferred@287539354 after retry RPC@1344178849)), so the following Deferred is getting paused: Deferred@287539354(state=PAUSED, result=Deferred@1411449978, callback=org.apache.kudu.client.AsyncKuduClient$6@4971c8b -> wakeup thread pool-3-thread-56, errback=passthrough -> wakeup thread pool-3-thread-56) 01:26:47.855 [DEBUG - New I/O worker #507] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.856 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.685 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=660), Traces: [0ms] refreshing cache from master, [287ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [287ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [288ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [632ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.860 [DEBUG - New I/O worker #239] (Deferred.java:1330) callback=retry RPC@1197319057 returned Deferred@1130508194(state=PENDING, result=null, callback=(continuation of Deferred@106998186 after retry RPC@1197319057), errback=(continuation of Deferred@106998186 after retry RPC@1197319057)), so the following Deferred is getting paused: Deferred@106998186(state=PAUSED, result=Deferred@1130508194, callback=org.apache.kudu.client.AsyncKuduClient$6@1dc7c160 -> wakeup thread pool-3-thread-41, errback=passthrough -> wakeup thread pool-3-thread-41) 01:26:47.683 [DEBUG - New I/O worker #409] (Deferred.java:1330) callback=retry RPC@2084609921 returned Deferred@2021594321(state=PENDING, result=null, callback=(continuation of Deferred@1031640506 after retry RPC@2084609921), errback=(continuation of Deferred@1031640506 after retry RPC@2084609921)), so the following Deferred is getting paused: Deferred@1031640506(state=PAUSED, result=Deferred@2021594321, callback=org.apache.kudu.client.AsyncKuduClient$6@74ce9fc3 -> wakeup thread pool-3-thread-64, errback=passthrough -> wakeup thread pool-3-thread-64) 01:26:47.862 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.681 [DEBUG - New I/O worker #439] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.678 [DEBUG - New I/O worker #408] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.676 [DEBUG - New I/O worker #535] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:47.668 [DEBUG - New I/O worker #508] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.869 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.869 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.870 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.870 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.870 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.871 [DEBUG - New I/O worker #413] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.871 [DEBUG - New I/O worker #413] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.872 [DEBUG - pool-3-thread-58] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.873 [DEBUG - New I/O worker #492] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.664 [DEBUG - New I/O worker #479] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:47.662 [DEBUG - New I/O worker #111] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.662 [DEBUG - New I/O worker #143] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.878 [DEBUG - New I/O worker #143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.878 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.879 [DEBUG - New I/O worker #155] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.880 [DEBUG - New I/O worker #188] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.877 [DEBUG - New I/O worker #111] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.873 [DEBUG - New I/O worker #417] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.870 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.882 [DEBUG - New I/O worker #436] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.883 [DEBUG - New I/O worker #111] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.869 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.928 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.929 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.929 [DEBUG - New I/O worker #353] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.930 [DEBUG - New I/O worker #353] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.865 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.931 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.853 [DEBUG - New I/O worker #71] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.932 [DEBUG - New I/O worker #71] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.933 [DEBUG - New I/O worker #418] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.933 [DEBUG - pool-3-thread-21] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.852 [DEBUG - New I/O worker #570] (Negotiator.java:764) Authenticated connection [id: 0xe5eea5cf, /127.1.193.126:51028 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.852 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.935 [DEBUG - New I/O worker #535] (Negotiator.java:764) Authenticated connection [id: 0xa3f20c07, /127.1.193.126:51036 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.849 [DEBUG - New I/O worker #391] (Negotiator.java:764) Authenticated connection [id: 0x2cd9dbe8, /127.1.193.124:46209 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.849 [DEBUG - New I/O worker #531] (Negotiator.java:764) Authenticated connection [id: 0x49702b94, /127.1.193.126:51039 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.849 [DEBUG - New I/O worker #475] (Negotiator.java:764) Authenticated connection [id: 0x442fa75a, /127.1.193.124:46228 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.847 [DEBUG - New I/O worker #527] (Negotiator.java:764) Authenticated connection [id: 0xa4908916, /127.1.193.126:51048 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.846 [DEBUG - New I/O worker #310] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.846 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.939 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.939 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.940 [DEBUG - New I/O worker #101] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.940 [DEBUG - New I/O worker #101] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.846 [DEBUG - New I/O worker #406] (Negotiator.java:764) Authenticated connection [id: 0x2dc1b98f, /127.1.193.125:50290 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.844 [DEBUG - New I/O worker #24] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.838 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.838 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.943 [DEBUG - pool-3-thread-25] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.837 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.836 [DEBUG - New I/O worker #435] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:47.836 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.808 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.946 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.946 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.946 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.947 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.947 [DEBUG - New I/O worker #356] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.948 [DEBUG - New I/O worker #356] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.949 [DEBUG - pool-3-thread-53] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.949 [DEBUG - New I/O worker #397] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.807 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.950 [DEBUG - New I/O worker #71] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.950 [DEBUG - New I/O worker #101] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.807 [DEBUG - New I/O worker #229] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.807 [DEBUG - New I/O worker #187] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.807 [DEBUG - New I/O worker #137] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.807 [DEBUG - New I/O worker #62] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.807 [DEBUG - pool-3-thread-32] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.807 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.952 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.952 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.953 [DEBUG - New I/O worker #102] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.953 [DEBUG - New I/O worker #102] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.806 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.954 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.954 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.955 [DEBUG - New I/O worker #11] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.955 [DEBUG - New I/O worker #11] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.956 [DEBUG - pool-3-thread-3] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.956 [DEBUG - New I/O worker #28] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.957 [DEBUG - New I/O worker #146] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.957 [DEBUG - New I/O worker #43] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.806 [DEBUG - New I/O worker #161] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.806 [DEBUG - New I/O worker #222] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.806 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.805 [DEBUG - New I/O worker #138] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.959 [DEBUG - New I/O worker #102] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.805 [DEBUG - New I/O worker #131] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.805 [DEBUG - New I/O worker #61] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.805 [DEBUG - pool-3-thread-35] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.805 [DEBUG - New I/O worker #179] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.805 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.961 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.804 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.964 [DEBUG - New I/O worker #539] (Negotiator.java:764) Authenticated connection [id: 0x46a034ba, /127.1.193.126:51030 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.965 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.804 [DEBUG - New I/O worker #77] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.804 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.966 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.804 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.967 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.967 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.968 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.968 [DEBUG - New I/O worker #411] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.969 [DEBUG - New I/O worker #411] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.969 [DEBUG - New I/O worker #508] (Negotiator.java:764) Authenticated connection [id: 0x5e66746b, /127.1.193.124:46219 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.801 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.801 [DEBUG - New I/O worker #228] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.801 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.970 [DEBUG - New I/O worker #490] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.971 [DEBUG - New I/O worker #542] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.971 [DEBUG - New I/O worker #353] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.801 [DEBUG - New I/O worker #185] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.800 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.800 [DEBUG - pool-3-thread-31] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.800 [DEBUG - New I/O worker #561] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - New I/O worker #387] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - New I/O worker #510] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - New I/O worker #347] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - New I/O worker #442] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - New I/O worker #437] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - Time-limited test] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.974 [DEBUG - New I/O worker #389] (Negotiator.java:764) Authenticated connection [id: 0xc4884347, /127.1.193.125:50300 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.799 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.975 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=951), Traces: [0ms] refreshing cache from master, [290ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [290ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [290ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [776ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.976 [DEBUG - New I/O worker #495] (Negotiator.java:764) Authenticated connection [id: 0x335af5a9, /127.1.193.126:51051 => /127.1.193.126:40739] using SASL/PLAIN 01:26:47.799 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.977 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=925), Traces: [0ms] refreshing cache from master, [405ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [405ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [406ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [608ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [748ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [757ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:47.977 [DEBUG - New I/O worker #307] (Deferred.java:1330) callback=retry RPC@1161640865 returned Deferred@1090693086(state=PENDING, result=null, callback=(continuation of Deferred@67121562 after retry RPC@1161640865), errback=(continuation of Deferred@67121562 after retry RPC@1161640865)), so the following Deferred is getting paused: Deferred@67121562(state=PAUSED, result=Deferred@1090693086, callback=org.apache.kudu.client.AsyncKuduClient$6@19020208 -> wakeup thread pool-3-thread-47, errback=passthrough -> wakeup thread pool-3-thread-47) 01:26:47.799 [DEBUG - pool-3-thread-50] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.799 [DEBUG - New I/O worker #342] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.799 [DEBUG - New I/O worker #303] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.798 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.979 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.979 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.980 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.980 [DEBUG - New I/O worker #349] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:47.980 [DEBUG - New I/O worker #349] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.798 [DEBUG - New I/O worker #105] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.798 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.981 [DEBUG - pool-3-thread-48] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.798 [DEBUG - pool-3-thread-45] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.982 [DEBUG - New I/O worker #481] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.798 [DEBUG - New I/O worker #150] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.798 [DEBUG - New I/O worker #49] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.798 [DEBUG - New I/O worker #109] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.798 [DEBUG - New I/O worker #74] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.983 [DEBUG - New I/O worker #507] (Negotiator.java:764) Authenticated connection [id: 0x1a3eab08, /127.1.193.124:46234 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.798 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.797 [DEBUG - pool-3-thread-20] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.797 [DEBUG - New I/O worker #51] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - pool-3-thread-19] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.797 [DEBUG - New I/O worker #302] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - New I/O worker #216] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - New I/O worker #53] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - New I/O worker #44] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - New I/O worker #18] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.987 [DEBUG - New I/O worker #349] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - New I/O worker #13] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - pool-3-thread-40] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.797 [DEBUG - New I/O worker #398] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.797 [DEBUG - New I/O worker #256] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #199] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #65] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #186] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - pool-3-thread-4] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.796 [DEBUG - New I/O worker #139] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #399] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - pool-3-thread-69] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.990 [DEBUG - New I/O worker #435] (Negotiator.java:764) Authenticated connection [id: 0xbfdcf72d, /127.1.193.125:50309 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.796 [DEBUG - New I/O worker #396] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #20] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - pool-3-thread-30] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.796 [DEBUG - New I/O worker #17] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #225] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #12] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #134] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - pool-3-thread-34] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.796 [DEBUG - New I/O worker #184] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.796 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.795 [DEBUG - New I/O worker #37] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - pool-3-thread-2] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.795 [DEBUG - New I/O worker #38] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - New I/O worker #76] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - New I/O worker #52] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - pool-3-thread-15] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.795 [DEBUG - New I/O worker #39] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - New I/O worker #27] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - New I/O worker #35] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - pool-3-thread-8] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.795 [DEBUG - pool-3-thread-13] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.004 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.795 [DEBUG - New I/O worker #48] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.795 [DEBUG - New I/O worker #148] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #106] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #255] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - pool-3-thread-37] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.794 [DEBUG - New I/O worker #218] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #176] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #149] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #181] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #21] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #50] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - pool-3-thread-27] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.794 [DEBUG - New I/O worker #502] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.794 [DEBUG - New I/O worker #469] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - New I/O worker #332] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - New I/O worker #159] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - New I/O worker #136] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - pool-3-thread-6] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.793 [DEBUG - New I/O worker #25] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - New I/O worker #513] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - New I/O worker #15] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.793 [DEBUG - pool-3-thread-33] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.793 [DEBUG - New I/O worker #383] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.792 [DEBUG - pool-3-thread-5] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.792 [DEBUG - New I/O worker #480] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.792 [DEBUG - New I/O worker #394] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.792 [DEBUG - pool-3-thread-74] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.792 [DEBUG - pool-3-thread-68] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.792 [DEBUG - New I/O worker #92] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.792 [DEBUG - New I/O worker #60] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.792 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.792 [DEBUG - New I/O worker #147] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.171 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.791 [DEBUG - New I/O worker #68] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.791 [DEBUG - New I/O worker #325] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.791 [DEBUG - New I/O worker #86] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.791 [DEBUG - New I/O worker #41] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.791 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.791 [DEBUG - New I/O worker #273] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.172 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1101), Traces: [0ms] refreshing cache from master, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [172ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [519ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.173 [DEBUG - New I/O worker #393] (Deferred.java:1330) callback=retry RPC@524871769 returned Deferred@460812136(state=PENDING, result=null, callback=(continuation of Deferred@1584783970 after retry RPC@524871769), errback=(continuation of Deferred@1584783970 after retry RPC@524871769)), so the following Deferred is getting paused: Deferred@1584783970(state=PAUSED, result=Deferred@460812136, callback=org.apache.kudu.client.AsyncKuduClient$6@37338270 -> wakeup thread pool-3-thread-70, errback=passthrough -> wakeup thread pool-3-thread-70) 01:26:47.791 [DEBUG - pool-3-thread-43] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.791 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.791 [DEBUG - New I/O worker #157] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.791 [DEBUG - New I/O worker #117] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.174 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.175 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.175 [DEBUG - New I/O worker #10] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.176 [DEBUG - New I/O worker #10] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.176 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.171 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.064 [DEBUG - New I/O worker #486] (Negotiator.java:764) Authenticated connection [id: 0xf78373ca, /127.1.193.124:46216 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.051 [DEBUG - New I/O worker #392] (Negotiator.java:764) Authenticated connection [id: 0xca4add0f, /127.1.193.125:50297 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.044 [DEBUG - New I/O worker #438] (Negotiator.java:764) Authenticated connection [id: 0xe54b44fe, /127.1.193.125:50294 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.040 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.010 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.005 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.182 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.182 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.183 [DEBUG - New I/O worker #404] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.183 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.184 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1169), Traces: [0ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [948ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1169ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.004 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.184 [DEBUG - New I/O worker #392] (Deferred.java:1330) callback=retry RPC@902468675 returned Deferred@838145570(state=PENDING, result=null, callback=(continuation of Deferred@1962385016 after retry RPC@902468675), errback=(continuation of Deferred@1962385016 after retry RPC@902468675)), so the following Deferred is getting paused: Deferred@1962385016(state=PAUSED, result=Deferred@838145570, callback=org.apache.kudu.client.AsyncKuduClient$6@24c09a59 -> wakeup thread pool-3-thread-79, errback=passthrough -> wakeup thread pool-3-thread-79) 01:26:48.002 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.995 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.185 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1173), Traces: [0ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [967ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [977ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.186 [DEBUG - New I/O worker #435] (Deferred.java:1330) callback=retry RPC@1945109437 returned Deferred@2010089755(state=PENDING, result=null, callback=(continuation of Deferred@852298118 after retry RPC@1945109437), errback=(continuation of Deferred@852298118 after retry RPC@1945109437)), so the following Deferred is getting paused: Deferred@852298118(state=PAUSED, result=Deferred@2010089755, callback=org.apache.kudu.client.AsyncKuduClient$6@70ab910e -> wakeup thread pool-3-thread-77, errback=passthrough -> wakeup thread pool-3-thread-77) 01:26:47.992 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.990 [DEBUG - New I/O worker #439] (Negotiator.java:764) Authenticated connection [id: 0x7749b6bd, /127.1.193.125:50303 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.988 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.986 [DEBUG - New I/O worker #356] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.187 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1173), Traces: [0ms] refreshing cache from master, [363ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [364ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [364ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [942ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [974ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:47.984 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=966), Traces: [0ms] refreshing cache from master, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:48.188 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.983 [DEBUG - New I/O worker #391] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.980 [DEBUG - New I/O worker #408] (Negotiator.java:764) Authenticated connection [id: 0x5d86b6dd, /127.1.193.124:46208 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.978 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:47.976 [DEBUG - New I/O worker #386] (Deferred.java:1330) callback=retry RPC@1927090611 returned Deferred@1994558256(state=PENDING, result=null, callback=(continuation of Deferred@870325128 after retry RPC@1927090611), errback=(continuation of Deferred@870325128 after retry RPC@1927090611)), so the following Deferred is getting paused: Deferred@870325128(state=PAUSED, result=Deferred@1994558256, callback=org.apache.kudu.client.AsyncKuduClient$6@1853f554 -> wakeup thread pool-3-thread-73, errback=passthrough -> wakeup thread pool-3-thread-73) 01:26:48.189 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:47.976 [DEBUG - New I/O worker #388] (Negotiator.java:764) Authenticated connection [id: 0x82a015d4, /127.1.193.124:46204 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.190 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:47.973 [DEBUG - New I/O worker #411] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.972 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.191 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.191 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.971 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.253 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.254 [DEBUG - New I/O worker #145] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.254 [DEBUG - New I/O worker #145] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.970 [DEBUG - pool-3-thread-65] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.967 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.967 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.966 [DEBUG - New I/O worker #479] (Negotiator.java:764) Authenticated connection [id: 0xea2d24e2, /127.1.193.124:46213 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.966 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.256 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.256 [DEBUG - New I/O worker #355] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.257 [DEBUG - New I/O worker #355] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.258 [DEBUG - pool-3-thread-28] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.962 [DEBUG - New I/O worker #11] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.259 [DEBUG - New I/O worker #430] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.962 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.260 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.260 [DEBUG - New I/O worker #308] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.260 [DEBUG - New I/O worker #308] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.961 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.261 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.262 [DEBUG - pool-3-thread-51] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.262 [DEBUG - New I/O worker #380] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.958 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.958 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.263 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.263 [DEBUG - New I/O worker #355] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.957 [DEBUG - New I/O worker #192] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.956 [DEBUG - pool-3-thread-24] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.264 [DEBUG - New I/O worker #145] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.950 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:47.949 [DEBUG - New I/O worker #431] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.946 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.265 [DEBUG - New I/O worker #258] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.265 [DEBUG - New I/O worker #258] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.943 [DEBUG - New I/O worker #144] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.943 [DEBUG - New I/O worker #154] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.266 [DEBUG - pool-3-thread-44] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.934 [DEBUG - New I/O worker #410] (Negotiator.java:764) Authenticated connection [id: 0x84bc7dc7, /127.1.193.125:50312 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.267 [DEBUG - New I/O worker #309] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.267 [DEBUG - New I/O worker #343] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.933 [DEBUG - New I/O worker #116] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.933 [DEBUG - New I/O worker #78] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.933 [DEBUG - New I/O worker #493] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:47.932 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:47.931 [DEBUG - pool-3-thread-46] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.927 [DEBUG - New I/O worker #476] (Negotiator.java:764) Authenticated connection [id: 0x83b92f16, /127.1.193.124:46225 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.927 [DEBUG - New I/O worker #488] (Negotiator.java:764) Authenticated connection [id: 0x8db3f705, /127.1.193.124:46237 => /127.1.193.124:58247] using SASL/PLAIN 01:26:47.926 [DEBUG - New I/O worker #413] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.924 [DEBUG - New I/O worker #436] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299959, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299958, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299958, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.883 [DEBUG - New I/O worker #348] (Negotiator.java:764) Authenticated connection [id: 0x7c978c7b, /127.1.193.125:50279 => /127.1.193.125:37435] using SASL/PLAIN 01:26:47.883 [DEBUG - New I/O worker #182] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.271 [DEBUG - pool-3-thread-76] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.883 [DEBUG - New I/O worker #143] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:47.882 [DEBUG - New I/O worker #130] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:47.882 [DEBUG - pool-3-thread-36] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:47.879 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.272 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.272 [DEBUG - New I/O worker #409] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.273 [DEBUG - New I/O worker #409] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:47.879 [DEBUG - pool-3-thread-29] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.271 [DEBUG - New I/O worker #436] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.273 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.274 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1252), Traces: [0ms] refreshing cache from master, [310ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [310ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [310ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [573ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1171ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1251ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.274 [DEBUG - New I/O worker #409] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.274 [DEBUG - New I/O worker #348] (Deferred.java:1330) callback=retry RPC@917545945 returned Deferred@848241111(state=PENDING, result=null, callback=(continuation of Deferred@2005772770 after retry RPC@917545945), errback=(continuation of Deferred@2005772770 after retry RPC@917545945)), so the following Deferred is getting paused: Deferred@2005772770(state=PAUSED, result=Deferred@848241111, callback=org.apache.kudu.client.AsyncKuduClient$6@75adf7c6 -> wakeup thread pool-3-thread-49, errback=passthrough -> wakeup thread pool-3-thread-49) 01:26:48.275 [DEBUG - New I/O worker #412] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.271 [DEBUG - New I/O worker #526] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.271 [DEBUG - New I/O worker #509] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.270 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.276 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1259), Traces: [0ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [973ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1253ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:48.277 [DEBUG - New I/O worker #410] (Deferred.java:1330) callback=retry RPC@802570516 returned Deferred@736739382(state=PENDING, result=null, callback=(continuation of Deferred@1860913967 after retry RPC@802570516), errback=(continuation of Deferred@1860913967 after retry RPC@802570516)), so the following Deferred is getting paused: Deferred@1860913967(state=PAUSED, result=Deferred@736739382, callback=org.apache.kudu.client.AsyncKuduClient$6@428c962f -> wakeup thread pool-3-thread-66, errback=passthrough -> wakeup thread pool-3-thread-66) 01:26:48.269 [DEBUG - New I/O worker #258] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.269 [DEBUG - New I/O worker #416] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.264 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.278 [DEBUG - New I/O worker #416] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.263 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.263 [DEBUG - New I/O worker #308] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.262 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.279 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.262 [DEBUG - New I/O worker #361] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.280 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.262 [DEBUG - New I/O worker #403] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.281 [DEBUG - New I/O worker #403] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.259 [DEBUG - New I/O worker #503] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.282 [DEBUG - pool-3-thread-56] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.259 [DEBUG - New I/O worker #191] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.283 [DEBUG - New I/O worker #427] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.258 [DEBUG - pool-3-thread-59] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.258 [DEBUG - New I/O worker #153] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.256 [DEBUG - New I/O worker #175] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.255 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.284 [DEBUG - New I/O worker #407] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.285 [DEBUG - New I/O worker #407] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.246 [DEBUG - pool-3-thread-21] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.233 [DEBUG - pool-3-thread-14] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.230 [DEBUG - pool-3-thread-16] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.287 [DEBUG - New I/O worker #424] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.287 [DEBUG - New I/O worker #485] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.287 [DEBUG - New I/O worker #407] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.201 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.201 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.200 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.195 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.291 [DEBUG - New I/O worker #403] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.292 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1279), Traces: [0ms] refreshing cache from master, [369ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [369ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [369ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [855ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [944ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1183ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.194 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.192 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.192 [DEBUG - New I/O worker #238] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.293 [DEBUG - New I/O worker #238] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.189 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1171), Traces: [0ms] refreshing cache from master, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [961ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [967ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [1165ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:48.188 [DEBUG - New I/O worker #390] (Deferred.java:1330) callback=retry RPC@1415221334 returned Deferred@1348800005(state=PENDING, result=null, callback=(continuation of Deferred@359111277 after retry RPC@1415221334), errback=(continuation of Deferred@359111277 after retry RPC@1415221334)), so the following Deferred is getting paused: Deferred@359111277(state=PAUSED, result=Deferred@1348800005, callback=org.apache.kudu.client.AsyncKuduClient$6@2460f379 -> wakeup thread pool-3-thread-72, errback=passthrough -> wakeup thread pool-3-thread-72) 01:26:48.188 [DEBUG - New I/O worker #389] (Deferred.java:1330) callback=retry RPC@52578049 returned Deferred@119391165(state=PENDING, result=null, callback=(continuation of Deferred@1109347642 after retry RPC@52578049), errback=(continuation of Deferred@1109347642 after retry RPC@52578049)), so the following Deferred is getting paused: Deferred@1109347642(state=PAUSED, result=Deferred@119391165, callback=org.apache.kudu.client.AsyncKuduClient$6@621c2674 -> wakeup thread pool-3-thread-71, errback=passthrough -> wakeup thread pool-3-thread-71) 01:26:48.186 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.299 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=1282), Traces: [0ms] refreshing cache from master, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [832ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [972ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [1170ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.300 [DEBUG - New I/O worker #438] (Deferred.java:1330) callback=retry RPC@458573264 returned Deferred@527094370(state=PENDING, result=null, callback=(continuation of Deferred@1516785643 after retry RPC@458573264), errback=(continuation of Deferred@1516785643 after retry RPC@458573264)), so the following Deferred is getting paused: Deferred@1516785643(state=PAUSED, result=Deferred@527094370, callback=org.apache.kudu.client.AsyncKuduClient$6@5ab2279d -> wakeup thread pool-3-thread-80, errback=passthrough -> wakeup thread pool-3-thread-80) 01:26:48.300 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.185 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.300 [DEBUG - pool-3-thread-42] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.301 [DEBUG - New I/O worker #263] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.301 [DEBUG - New I/O worker #312] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.185 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.302 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.302 [DEBUG - New I/O worker #414] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.303 [DEBUG - New I/O worker #414] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.184 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.310 [DEBUG - pool-3-thread-57] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.183 [DEBUG - New I/O worker #404] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.182 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.310 [DEBUG - New I/O worker #541] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.311 [DEBUG - pool-3-thread-67] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.311 [DEBUG - New I/O worker #426] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.311 [DEBUG - New I/O worker #428] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.312 [DEBUG - New I/O worker #404] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.179 [DEBUG - New I/O worker #22] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.312 [DEBUG - New I/O worker #22] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.177 [DEBUG - New I/O worker #32] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.177 [DEBUG - pool-3-thread-1] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.313 [DEBUG - pool-3-thread-11] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.177 [DEBUG - New I/O worker #10] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.177 [DEBUG - New I/O worker #14] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.314 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.314 [DEBUG - New I/O worker #36] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.313 [DEBUG - New I/O worker #42] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.311 [DEBUG - New I/O worker #414] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.310 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.310 [DEBUG - New I/O worker #489] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.310 [DEBUG - New I/O worker #400] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.316 [DEBUG - New I/O worker #400] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.308 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.304 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.317 [DEBUG - New I/O worker #400] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.304 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.317 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.304 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.318 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.318 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.319 [DEBUG - New I/O worker #435] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.319 [DEBUG - New I/O worker #435] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.303 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.320 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.321 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.321 [DEBUG - New I/O worker #410] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.321 [DEBUG - New I/O worker #527] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.322 [DEBUG - New I/O worker #435] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.300 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.322 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.323 [DEBUG - New I/O worker #239] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.323 [DEBUG - New I/O worker #239] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.298 [DEBUG - New I/O worker #406] (Deferred.java:1330) callback=retry RPC@265988311 returned Deferred@199568884(state=PENDING, result=null, callback=(continuation of Deferred@1323803372 after retry RPC@265988311), errback=(continuation of Deferred@1323803372 after retry RPC@265988311)), so the following Deferred is getting paused: Deferred@1323803372(state=PAUSED, result=Deferred@199568884, callback=org.apache.kudu.client.AsyncKuduClient$6@25a5e4d2 -> wakeup thread pool-3-thread-61, errback=passthrough -> wakeup thread pool-3-thread-61) 01:26:48.324 [DEBUG - New I/O worker #260] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.324 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.298 [DEBUG - New I/O worker #238] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.297 [DEBUG - New I/O worker #439] (Deferred.java:1330) callback=retry RPC@1756867524 returned Deferred@1820860938(state=PENDING, result=null, callback=(continuation of Deferred@696951295 after retry RPC@1756867524), errback=(continuation of Deferred@696951295 after retry RPC@1756867524)), so the following Deferred is getting paused: Deferred@696951295(state=PAUSED, result=Deferred@1820860938, callback=org.apache.kudu.client.AsyncKuduClient$6@4dd5062d -> wakeup thread pool-3-thread-75, errback=passthrough -> wakeup thread pool-3-thread-75) 01:26:48.325 [DEBUG - New I/O worker #239] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.292 [DEBUG - New I/O worker #393] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.326 [DEBUG - New I/O worker #393] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.327 [DEBUG - pool-3-thread-70] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.291 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.327 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.327 [DEBUG - New I/O worker #392] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.291 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.328 [DEBUG - New I/O worker #392] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.328 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.328 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.329 [DEBUG - pool-3-thread-79] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.329 [DEBUG - New I/O worker #482] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.329 [DEBUG - New I/O worker #535] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.291 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.330 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.330 [DEBUG - New I/O worker #307] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.330 [DEBUG - New I/O worker #307] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.287 [DEBUG - pool-3-thread-60] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.331 [DEBUG - pool-3-thread-47] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.284 [DEBUG - New I/O worker #175] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.332 [DEBUG - New I/O worker #357] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.282 [DEBUG - New I/O worker #487] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.282 [DEBUG - New I/O worker #501] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.332 [DEBUG - New I/O worker #445] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.333 [DEBUG - pool-3-thread-38] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.282 [DEBUG - pool-3-thread-62] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.282 [DEBUG - New I/O worker #422] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.282 [DEBUG - New I/O worker #416] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.279 [DEBUG - New I/O worker #350] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.334 [DEBUG - New I/O worker #350] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.334 [DEBUG - New I/O worker #175] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.279 [DEBUG - New I/O worker #433] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.278 [DEBUG - New I/O worker #352] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.335 [DEBUG - New I/O worker #352] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.274 [DEBUG - New I/O worker #415] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.274 [DEBUG - pool-3-thread-64] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.335 [DEBUG - New I/O worker #432] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.335 [DEBUG - New I/O worker #433] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.337 [DEBUG - New I/O worker #420] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.337 [DEBUG - New I/O worker #491] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.338 [DEBUG - pool-3-thread-78] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.338 [DEBUG - New I/O worker #478] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.339 [DEBUG - New I/O worker #781] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.339 [DEBUG - New I/O worker #569] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.335 [DEBUG - New I/O worker #395] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.335 [DEBUG - pool-3-thread-52] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.333 [DEBUG - New I/O worker #257] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.340 [DEBUG - New I/O worker #741] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.333 [DEBUG - New I/O worker #217] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.332 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.331 [DEBUG - New I/O worker #307] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.330 [DEBUG - New I/O worker #392] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.330 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.342 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.342 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.343 [DEBUG - New I/O worker #406] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.343 [DEBUG - New I/O worker #406] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.329 [DEBUG - New I/O worker #393] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.345 [DEBUG - pool-3-thread-61] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.329 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.328 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.346 [DEBUG - New I/O worker #486] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.346 [DEBUG - New I/O worker #539] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.347 [DEBUG - New I/O worker #406] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.327 [DEBUG - New I/O worker #440] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.327 [DEBUG - New I/O worker #434] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.325 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.324 [DEBUG - New I/O worker #305] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.324 [DEBUG - pool-3-thread-41] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.322 [DEBUG - New I/O worker #507] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.321 [DEBUG - New I/O worker #410] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.321 [DEBUG - pool-3-thread-77] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.353 [DEBUG - pool-3-thread-66] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.354 [DEBUG - New I/O worker #410] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.318 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.355 [DEBUG - New I/O worker #348] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.355 [DEBUG - New I/O worker #348] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.356 [DEBUG - New I/O worker #488] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.356 [DEBUG - pool-3-thread-49] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.356 [DEBUG - New I/O worker #388] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.357 [DEBUG - New I/O worker #441] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.317 [DEBUG - New I/O worker #483] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.317 [DEBUG - New I/O worker #537] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.317 [DEBUG - pool-3-thread-63] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.316 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.315 [DEBUG - New I/O worker #22] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.315 [DEBUG - New I/O worker #354] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.359 [DEBUG - New I/O worker #354] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.360 [DEBUG - pool-3-thread-54] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.360 [DEBUG - New I/O worker #408] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.362 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.362 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.363 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.363 [DEBUG - New I/O worker #439] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.363 [DEBUG - New I/O worker #439] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.364 [DEBUG - pool-3-thread-75] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.364 [DEBUG - New I/O worker #475] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.365 [DEBUG - New I/O worker #439] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.358 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.365 [DEBUG - New I/O worker #390] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.357 [DEBUG - New I/O worker #348] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.353 [DEBUG - New I/O worker #495] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.348 [DEBUG - New I/O worker #781] (Negotiator.java:764) Authenticated connection [id: 0xaf991dab, /127.1.193.124:46240 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.346 [DEBUG - New I/O worker #386] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.346 [DEBUG - New I/O worker #438] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.367 [DEBUG - New I/O worker #386] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.342 [DEBUG - New I/O worker #352] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.341 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.368 [DEBUG - pool-3-thread-73] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.368 [DEBUG - New I/O worker #506] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.340 [DEBUG - New I/O worker #783] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.370 [DEBUG - New I/O worker #386] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.338 [DEBUG - New I/O worker #433] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.338 [DEBUG - New I/O worker #350] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.337 [DEBUG - pool-3-thread-55] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.368 [DEBUG - New I/O worker #474] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.368 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.405 [DEBUG - New I/O worker #389] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.405 [DEBUG - New I/O worker #389] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.367 [DEBUG - New I/O worker #438] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.366 [DEBUG - New I/O worker #390] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.365 [DEBUG - New I/O worker #354] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.364 [DEBUG - New I/O worker #532] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.407 [DEBUG - New I/O worker #531] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.407 [DEBUG - New I/O worker #476] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.408 [DEBUG - pool-3-thread-72] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.360 [DEBUG - New I/O worker #423] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.408 [DEBUG - New I/O worker #560] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.409 [DEBUG - New I/O worker #570] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.359 [DEBUG - New I/O worker #741] (Negotiator.java:764) Authenticated connection [id: 0xc6a4180c, /127.1.193.125:50315 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.408 [DEBUG - New I/O worker #508] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.408 [DEBUG - New I/O worker #479] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.408 [DEBUG - New I/O worker #390] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.411 [DEBUG - New I/O worker #438] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.411 [DEBUG - New I/O worker #389] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.411 [DEBUG - New I/O worker #783] (Negotiator.java:764) Authenticated connection [id: 0xdc53af91, /127.1.193.126:51054 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.407 [DEBUG - pool-3-thread-80] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.407 [DEBUG - pool-3-thread-71] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.412 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.413 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=126), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [119ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.414 [DEBUG - New I/O worker #741] (Deferred.java:1330) callback=retry RPC@1239740084 returned Deferred@1306223440(state=PENDING, result=null, callback=(continuation of Deferred@148497551 after retry RPC@1239740084), errback=(continuation of Deferred@148497551 after retry RPC@1239740084)), so the following Deferred is getting paused: Deferred@148497551(state=PAUSED, result=Deferred@1306223440, callback=org.apache.kudu.client.AsyncKuduClient$6@57648fbc -> wakeup thread pool-3-thread-14, errback=passthrough -> wakeup thread pool-3-thread-14) 01:26:48.416 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.420 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.421 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.421 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.421 [DEBUG - New I/O worker #741] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.422 [DEBUG - New I/O worker #741] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.423 [DEBUG - pool-3-thread-14] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.423 [DEBUG - New I/O worker #783] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.423 [DEBUG - New I/O worker #781] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.423 [DEBUG - New I/O worker #741] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.522 [DEBUG - pool-3-thread-31] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.523 [DEBUG - New I/O worker #746] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.543 [DEBUG - New I/O worker #750] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.549 [DEBUG - pool-3-thread-3] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.549 [DEBUG - pool-3-thread-32] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.555 [DEBUG - New I/O worker #746] (Negotiator.java:764) Authenticated connection [id: 0x878e8884, /127.1.193.124:46244 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.557 [DEBUG - pool-3-thread-22] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.559 [DEBUG - New I/O worker #739] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.560 [DEBUG - New I/O worker #754] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.562 [DEBUG - New I/O worker #752] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.562 [DEBUG - New I/O worker #769] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.565 [DEBUG - New I/O worker #750] (Negotiator.java:764) Authenticated connection [id: 0xa99bd1d2, /127.1.193.125:50319 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.568 [DEBUG - New I/O worker #748] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.572 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.573 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=286), Traces: [0ms] refreshing cache from master, [218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [260ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [283ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.577 [DEBUG - New I/O worker #797] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.581 [DEBUG - New I/O worker #752] (Negotiator.java:764) Authenticated connection [id: 0x67dfaf1d, /127.1.193.124:46245 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.583 [DEBUG - New I/O worker #796] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.584 [DEBUG - pool-3-thread-69] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.584 [DEBUG - New I/O worker #750] (Deferred.java:1330) callback=retry RPC@48475139 returned Deferred@115095687(state=PENDING, result=null, callback=(continuation of Deferred@1138665016 after retry RPC@48475139), errback=(continuation of Deferred@1138665016 after retry RPC@48475139)), so the following Deferred is getting paused: Deferred@1138665016(state=PAUSED, result=Deferred@115095687, callback=org.apache.kudu.client.AsyncKuduClient$6@6d4bee5f -> wakeup thread pool-3-thread-16, errback=passthrough -> wakeup thread pool-3-thread-16) 01:26:48.585 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.585 [DEBUG - New I/O worker #739] (Negotiator.java:764) Authenticated connection [id: 0x6a555572, /127.1.193.125:50318 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.588 [DEBUG - New I/O worker #754] (Negotiator.java:764) Authenticated connection [id: 0x63a77d38, /127.1.193.126:51059 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.577 [DEBUG - New I/O worker #769] (Negotiator.java:764) Authenticated connection [id: 0x5edfd41d, /127.1.193.125:50323 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.594 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.595 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.595 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.595 [DEBUG - New I/O worker #797] (Negotiator.java:764) Authenticated connection [id: 0x54f391b0, /127.1.193.126:51062 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.595 [DEBUG - New I/O worker #750] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.596 [DEBUG - New I/O worker #750] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.600 [DEBUG - pool-3-thread-16] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.600 [DEBUG - New I/O worker #754] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.600 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.600 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=77), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [76ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [77ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.601 [DEBUG - New I/O worker #769] (Deferred.java:1330) callback=retry RPC@1389548601 returned Deferred@1458395835(state=PENDING, result=null, callback=(continuation of Deferred@334487042 after retry RPC@1389548601), errback=(continuation of Deferred@334487042 after retry RPC@1389548601)), so the following Deferred is getting paused: Deferred@334487042(state=PAUSED, result=Deferred@1458395835, callback=org.apache.kudu.client.AsyncKuduClient$6@66012edd -> wakeup thread pool-3-thread-31, errback=passthrough -> wakeup thread pool-3-thread-31) 01:26:48.597 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.596 [DEBUG - New I/O worker #796] (Negotiator.java:764) Authenticated connection [id: 0x5bf88fa9, /127.1.193.124:46248 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.602 [DEBUG - New I/O worker #750] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.600 [DEBUG - New I/O worker #752] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.603 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=317), Traces: [0ms] refreshing cache from master, [217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [273ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.604 [DEBUG - New I/O worker #739] (Deferred.java:1330) callback=retry RPC@527799315 returned Deferred@457835301(state=PENDING, result=null, callback=(continuation of Deferred@1581817384 after retry RPC@527799315), errback=(continuation of Deferred@1581817384 after retry RPC@527799315)), so the following Deferred is getting paused: Deferred@1581817384(state=PAUSED, result=Deferred@457835301, callback=org.apache.kudu.client.AsyncKuduClient$6@5c3ab574 -> wakeup thread pool-3-thread-21, errback=passthrough -> wakeup thread pool-3-thread-21) 01:26:48.607 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.607 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.613 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.613 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.613 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.614 [DEBUG - New I/O worker #739] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.614 [DEBUG - New I/O worker #739] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.615 [DEBUG - pool-3-thread-21] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.616 [DEBUG - New I/O worker #746] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.616 [DEBUG - New I/O worker #739] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:48.618949 3861 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.552721 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51063 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.566591 (+ 13870us) server_negotiation.cc:184] Beginning negotiation 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.566622 (+ 31us) server_negotiation.cc:373] Waiting for connection header 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.566678 (+ 56us) server_negotiation.cc:381] Connection header received 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.567093 (+ 415us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.567128 (+ 35us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.567289 (+ 161us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.570207 (+ 2918us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.570585 (+ 378us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.572323 (+ 1738us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:48.616926 (+ 44603us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51063: BlockingRecv error: recv got EOF from 127.1.193.126:51063 (error 108) 01:26:48.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":13609,"thread_start_us":14389,"threads_started":1} 01:26:48.621 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.622 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.622 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.622 [DEBUG - New I/O worker #769] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.623 [DEBUG - New I/O worker #769] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.624 [DEBUG - pool-3-thread-31] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.624 [DEBUG - New I/O worker #796] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.624 [DEBUG - New I/O worker #769] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.624 [DEBUG - New I/O worker #797] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.653 [DEBUG - pool-3-thread-10] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.703 [INFO - New I/O worker #748] (Connection.java:435) [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:48.703 [DEBUG - New I/O worker #748] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:48.704 [INFO - New I/O worker #748] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:48.710 [DEBUG - New I/O worker #758] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.713 [DEBUG - New I/O worker #840] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.717 [DEBUG - New I/O worker #773] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.718 [DEBUG - New I/O worker #822] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.718 [DEBUG - New I/O worker #807] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.718 [DEBUG - New I/O worker #757] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.718 [DEBUG - New I/O worker #855] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.720 [DEBUG - New I/O worker #794] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.727 [DEBUG - pool-3-thread-18] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.731 [DEBUG - New I/O worker #780] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.732 [DEBUG - New I/O worker #821] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.734 [DEBUG - New I/O worker #790] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.735 [DEBUG - New I/O worker #775] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.737 [DEBUG - New I/O worker #740] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.738 [DEBUG - New I/O worker #772] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.751 [DEBUG - New I/O worker #840] (Negotiator.java:764) Authenticated connection [id: 0x7c7d5dcb, /127.1.193.126:51069 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.754 [DEBUG - New I/O worker #855] (Negotiator.java:764) Authenticated connection [id: 0xf4b98bdb, /127.1.193.126:51072 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.756 [DEBUG - New I/O worker #773] (Negotiator.java:764) Authenticated connection [id: 0xa44b0c75, /127.1.193.125:50328 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.756 [DEBUG - New I/O worker #757] (Negotiator.java:764) Authenticated connection [id: 0xd56076f1, /127.1.193.125:50327 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.762 [DEBUG - New I/O worker #807] (Negotiator.java:764) Authenticated connection [id: 0x5d5c5a83, /127.1.193.124:46258 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.765 [DEBUG - New I/O worker #821] (Negotiator.java:764) Authenticated connection [id: 0x64dd8abd, /127.1.193.124:46253 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.765 [DEBUG - New I/O worker #790] (Negotiator.java:764) Authenticated connection [id: 0x87f10fad, /127.1.193.126:51075 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.767 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.767 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=182), Traces: [0ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [172ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.768 [DEBUG - New I/O worker #773] (Deferred.java:1330) callback=retry RPC@540030863 returned Deferred@605005649(state=PENDING, result=null, callback=(continuation of Deferred@1628257716 after retry RPC@540030863), errback=(continuation of Deferred@1628257716 after retry RPC@540030863)), so the following Deferred is getting paused: Deferred@1628257716(state=PAUSED, result=Deferred@605005649, callback=org.apache.kudu.client.AsyncKuduClient$6@710b7370 -> wakeup thread pool-3-thread-69, errback=passthrough -> wakeup thread pool-3-thread-69) 01:26:48.770 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.770 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=116), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [117ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [117ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:48.771 [DEBUG - New I/O worker #758] (Negotiator.java:764) Authenticated connection [id: 0x56877c92, /127.1.193.125:50333 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.771 [DEBUG - New I/O worker #757] (Deferred.java:1330) callback=retry RPC@1135231315 returned Deferred@1201004030(state=PENDING, result=null, callback=(continuation of Deferred@43469672 after retry RPC@1135231315), errback=(continuation of Deferred@43469672 after retry RPC@1135231315)), so the following Deferred is getting paused: Deferred@43469672(state=PAUSED, result=Deferred@1201004030, callback=org.apache.kudu.client.AsyncKuduClient$6@226e4472 -> wakeup thread pool-3-thread-10, errback=passthrough -> wakeup thread pool-3-thread-10) 01:26:48.773 [DEBUG - New I/O worker #794] (Negotiator.java:764) Authenticated connection [id: 0x6b285048, /127.1.193.124:46254 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.774 [DEBUG - New I/O worker #740] (Negotiator.java:764) Authenticated connection [id: 0x7e3ee2de, /127.1.193.125:50336 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.774 [DEBUG - New I/O worker #822] (Negotiator.java:764) Authenticated connection [id: 0x0118668f, /127.1.193.126:51068 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.775 [DEBUG - New I/O worker #775] (Negotiator.java:764) Authenticated connection [id: 0xc9f5bc78, /127.1.193.126:51078 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.778 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.778 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=228), Traces: [0ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [224ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [227ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.779 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.779 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=221), Traces: [0ms] refreshing cache from master, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [205ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [215ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.779 [DEBUG - New I/O worker #768] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.780 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.779 [DEBUG - New I/O worker #780] (Negotiator.java:764) Authenticated connection [id: 0xb02a142a, /127.1.193.124:46261 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.779 [DEBUG - New I/O worker #740] (Deferred.java:1330) callback=retry RPC@1908206691 returned Deferred@1971545740(state=PENDING, result=null, callback=(continuation of Deferred@813819480 after retry RPC@1908206691), errback=(continuation of Deferred@813819480 after retry RPC@1908206691)), so the following Deferred is getting paused: Deferred@813819480(state=PAUSED, result=Deferred@1971545740, callback=org.apache.kudu.client.AsyncKuduClient$6@17b21959 -> wakeup thread pool-3-thread-3, errback=passthrough -> wakeup thread pool-3-thread-3) 01:26:48.781 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.779 [DEBUG - New I/O worker #758] (Deferred.java:1330) callback=retry RPC@108593139 returned Deferred@38239204(state=PENDING, result=null, callback=(continuation of Deferred@1195767240 after retry RPC@108593139), errback=(continuation of Deferred@1195767240 after retry RPC@108593139)), so the following Deferred is getting paused: Deferred@1195767240(state=PAUSED, result=Deferred@38239204, callback=org.apache.kudu.client.AsyncKuduClient$6@370e7604 -> wakeup thread pool-3-thread-22, errback=passthrough -> wakeup thread pool-3-thread-22) 01:26:48.784 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.786 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.786 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.787 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.789 [DEBUG - New I/O worker #773] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.789 [DEBUG - New I/O worker #773] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.790 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.791 [DEBUG - pool-3-thread-20] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.791 [DEBUG - New I/O worker #812] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.788 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.792 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.792 [DEBUG - New I/O worker #747] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.793 [DEBUG - New I/O worker #794] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.798 [DEBUG - New I/O worker #811] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.801 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.801 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.801 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.802 [DEBUG - New I/O worker #740] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.802 [DEBUG - New I/O worker #740] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.803 [DEBUG - New I/O worker #768] (Negotiator.java:764) Authenticated connection [id: 0xaf235d9e, /127.1.193.125:50339 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.803 [DEBUG - New I/O worker #812] (Negotiator.java:764) Authenticated connection [id: 0x0be58226, /127.1.193.126:51081 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.803 [DEBUG - New I/O worker #790] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.803 [DEBUG - New I/O worker #740] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.804 [DEBUG - New I/O worker #773] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.807 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.788 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.803 [DEBUG - New I/O worker #780] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.803 [DEBUG - pool-3-thread-3] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.792 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.790 [DEBUG - New I/O worker #840] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.809 [DEBUG - New I/O worker #758] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.790 [DEBUG - pool-3-thread-69] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.809 [DEBUG - New I/O worker #758] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.810 [DEBUG - pool-3-thread-22] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.810 [DEBUG - New I/O worker #758] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.811 [DEBUG - New I/O worker #807] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.811 [DEBUG - New I/O worker #855] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.809 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.811 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=261), Traces: [0ms] refreshing cache from master, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [232ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [258ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.812 [DEBUG - New I/O worker #768] (Deferred.java:1330) callback=retry RPC@37447188 returned Deferred@100987531(state=PENDING, result=null, callback=(continuation of Deferred@1124494383 after retry RPC@37447188), errback=(continuation of Deferred@1124494383 after retry RPC@37447188)), so the following Deferred is getting paused: Deferred@1124494383(state=PAUSED, result=Deferred@100987531, callback=org.apache.kudu.client.AsyncKuduClient$6@2bdd06f5 -> wakeup thread pool-3-thread-32, errback=passthrough -> wakeup thread pool-3-thread-32) 01:26:48.812 [DEBUG - New I/O worker #772] (Negotiator.java:764) Authenticated connection [id: 0xf49e2555, /127.1.193.124:46264 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.808 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.813 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.813 [DEBUG - New I/O worker #757] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.814 [DEBUG - New I/O worker #757] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.815 [DEBUG - pool-3-thread-10] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.815 [DEBUG - New I/O worker #822] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.815 [DEBUG - New I/O worker #757] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.816 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.816 [DEBUG - New I/O worker #821] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.816 [DEBUG - New I/O worker #811] (Negotiator.java:764) Authenticated connection [id: 0x7a04c5d4, /127.1.193.124:46267 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.816 [DEBUG - New I/O worker #747] (Negotiator.java:764) Authenticated connection [id: 0xcb1facab, /127.1.193.125:50342 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.819 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.819 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.820 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.820 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.820 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=92), Traces: [0ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [81ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [91ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [91ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:48.820 [DEBUG - New I/O worker #768] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.821 [DEBUG - New I/O worker #747] (Deferred.java:1330) callback=retry RPC@511409650 returned Deferred@440728293(state=PENDING, result=null, callback=(continuation of Deferred@1598453705 after retry RPC@511409650), errback=(continuation of Deferred@1598453705 after retry RPC@511409650)), so the following Deferred is getting paused: Deferred@1598453705(state=PAUSED, result=Deferred@440728293, callback=org.apache.kudu.client.AsyncKuduClient$6@2b02758d -> wakeup thread pool-3-thread-18, errback=passthrough -> wakeup thread pool-3-thread-18) 01:26:48.821 [DEBUG - New I/O worker #768] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.822 [DEBUG - pool-3-thread-32] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.822 [DEBUG - New I/O worker #775] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.822 [DEBUG - New I/O worker #772] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.823 [DEBUG - New I/O worker #768] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.828 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.832 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.832 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.832 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.833 [DEBUG - New I/O worker #747] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.833 [DEBUG - New I/O worker #747] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.834 [DEBUG - pool-3-thread-18] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.834 [DEBUG - New I/O worker #747] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.834 [DEBUG - New I/O worker #811] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.835 [DEBUG - New I/O worker #812] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.852 [DEBUG - pool-3-thread-58] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.852 [DEBUG - pool-3-thread-9] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.884 [DEBUG - pool-3-thread-25] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.889 [DEBUG - pool-3-thread-39] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.892 [DEBUG - New I/O worker #763] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.892 [DEBUG - New I/O worker #826] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.892 [DEBUG - New I/O worker #827] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.897 [DEBUG - New I/O worker #831] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.898 [DEBUG - New I/O worker #778] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.898 [DEBUG - New I/O worker #791] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.899 [DEBUG - pool-3-thread-7] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.900 [DEBUG - pool-3-thread-17] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.902 [DEBUG - pool-3-thread-12] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.904 [DEBUG - New I/O worker #763] (Negotiator.java:764) Authenticated connection [id: 0xfce4ce9a, /127.1.193.125:50345 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.907 [DEBUG - New I/O worker #831] (Negotiator.java:764) Authenticated connection [id: 0xbcc56651, /127.1.193.126:51087 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.908 [DEBUG - New I/O worker #827] (Negotiator.java:764) Authenticated connection [id: 0x169b191c, /127.1.193.126:51084 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.909 [DEBUG - New I/O worker #791] (Negotiator.java:764) Authenticated connection [id: 0x6a453119, /127.1.193.124:46273 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.910 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.910 [DEBUG - pool-3-thread-26] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.912 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=59), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [56ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.913 [DEBUG - New I/O worker #763] (Deferred.java:1330) callback=retry RPC@929477081 returned Deferred@861479917(state=PENDING, result=null, callback=(continuation of Deferred@1985723362 after retry RPC@929477081), errback=(continuation of Deferred@1985723362 after retry RPC@929477081)), so the following Deferred is getting paused: Deferred@1985723362(state=PAUSED, result=Deferred@861479917, callback=org.apache.kudu.client.AsyncKuduClient$6@458de014 -> wakeup thread pool-3-thread-58, errback=passthrough -> wakeup thread pool-3-thread-58) 01:26:48.917 [DEBUG - New I/O worker #742] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.917 [DEBUG - New I/O worker #810] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.917 [DEBUG - New I/O worker #743] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.919 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.922 [DEBUG - New I/O worker #778] (Negotiator.java:764) Authenticated connection [id: 0x7881334b, /127.1.193.125:50348 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.923 [DEBUG - New I/O worker #826] (Negotiator.java:764) Authenticated connection [id: 0x3ef1e21a, /127.1.193.124:46270 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.921 [DEBUG - New I/O worker #867] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.921 [DEBUG - New I/O worker #829] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.927 [DEBUG - New I/O worker #742] (Negotiator.java:764) Authenticated connection [id: 0x827e1b28, /127.1.193.125:50351 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.921 [DEBUG - New I/O worker #771] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.930 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.931 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=123), Traces: [0ms] refreshing cache from master, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [122ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.932 [DEBUG - New I/O worker #742] (Deferred.java:1330) callback=retry RPC@1929610570 returned Deferred@2000429042(state=PENDING, result=null, callback=(continuation of Deferred@842960753 after retry RPC@1929610570), errback=(continuation of Deferred@842960753 after retry RPC@1929610570)), so the following Deferred is getting paused: Deferred@842960753(state=PAUSED, result=Deferred@2000429042, callback=org.apache.kudu.client.AsyncKuduClient$6@6c009dd0 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:26:48.933 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.937 [DEBUG - New I/O worker #829] (Negotiator.java:764) Authenticated connection [id: 0x9ee5dcd0, /127.1.193.124:46279 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.938 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.939 [DEBUG - New I/O worker #810] (Negotiator.java:764) Authenticated connection [id: 0x8935d4da, /127.1.193.126:51090 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.939 [DEBUG - New I/O worker #771] (Negotiator.java:764) Authenticated connection [id: 0x76ff3533, /127.1.193.125:50354 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.932 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.940 [DEBUG - New I/O worker #743] (Negotiator.java:764) Authenticated connection [id: 0x9cfad929, /127.1.193.124:46276 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.937 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=84), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [58ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [80ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.942 [DEBUG - New I/O worker #778] (Deferred.java:1330) callback=retry RPC@78338866 returned Deferred@9752447(state=PENDING, result=null, callback=(continuation of Deferred@1167479049 after retry RPC@78338866), errback=(continuation of Deferred@1167479049 after retry RPC@78338866)), so the following Deferred is getting paused: Deferred@1167479049(state=PAUSED, result=Deferred@9752447, callback=org.apache.kudu.client.AsyncKuduClient$6@2ab5d968 -> wakeup thread pool-3-thread-9, errback=passthrough -> wakeup thread pool-3-thread-9) 01:26:48.937 [DEBUG - New I/O worker #867] (Negotiator.java:764) Authenticated connection [id: 0x14e579f8, /127.1.193.126:51093 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.944 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.944 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.945 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.945 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.945 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=153), Traces: [0ms] refreshing cache from master, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [149ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [153ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.946 [DEBUG - New I/O worker #771] (Deferred.java:1330) callback=retry RPC@991695955 returned Deferred@1059293160(state=PENDING, result=null, callback=(continuation of Deferred@2048990824 after retry RPC@991695955), errback=(continuation of Deferred@2048990824 after retry RPC@991695955)), so the following Deferred is getting paused: Deferred@2048990824(state=PAUSED, result=Deferred@1059293160, callback=org.apache.kudu.client.AsyncKuduClient$6@10749298 -> wakeup thread pool-3-thread-20, errback=passthrough -> wakeup thread pool-3-thread-20) 01:26:48.950 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.945 [DEBUG - New I/O worker #763] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.950 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.951 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.952 [DEBUG - pool-3-thread-23] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:48.955 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.955 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.956 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.956 [DEBUG - New I/O worker #771] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.956 [DEBUG - New I/O worker #771] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.957 [DEBUG - pool-3-thread-20] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.948 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.957 [DEBUG - New I/O worker #829] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.957 [DEBUG - New I/O worker #867] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.951 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.951 [DEBUG - New I/O worker #763] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.958 [DEBUG - New I/O worker #778] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.959 [DEBUG - New I/O worker #778] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.960 [DEBUG - pool-3-thread-9] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.960 [DEBUG - New I/O worker #778] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.961 [DEBUG - New I/O worker #831] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.962 [DEBUG - New I/O worker #808] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.962 [DEBUG - pool-3-thread-58] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.962 [DEBUG - New I/O worker #826] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.963 [DEBUG - New I/O worker #827] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.963 [DEBUG - New I/O worker #763] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.958 [DEBUG - New I/O worker #771] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.958 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.964 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.965 [DEBUG - New I/O worker #742] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.965 [DEBUG - New I/O worker #742] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.960 [DEBUG - New I/O worker #791] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.965 [DEBUG - New I/O worker #815] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.966 [DEBUG - Time-limited test] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.967 [DEBUG - New I/O worker #742] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.967 [DEBUG - New I/O worker #753] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.968 [DEBUG - New I/O worker #743] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.969 [DEBUG - New I/O worker #810] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.975 [DEBUG - New I/O worker #808] (Negotiator.java:764) Authenticated connection [id: 0xdfde5173, /127.1.193.124:46282 => /127.1.193.124:58247] using SASL/PLAIN 01:26:48.976 [DEBUG - New I/O worker #815] (Negotiator.java:764) Authenticated connection [id: 0xc5363584, /127.1.193.126:51096 => /127.1.193.126:40739] using SASL/PLAIN 01:26:48.978 [DEBUG - New I/O worker #753] (Negotiator.java:764) Authenticated connection [id: 0x127c7527, /127.1.193.125:50357 => /127.1.193.125:37435] using SASL/PLAIN 01:26:48.981 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:48.981 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=69), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [66ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [67ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [69ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:48.982 [DEBUG - New I/O worker #753] (Deferred.java:1330) callback=retry RPC@424593705 returned Deferred@493965016(state=PENDING, result=null, callback=(continuation of Deferred@1483981586 after retry RPC@424593705), errback=(continuation of Deferred@1483981586 after retry RPC@424593705)), so the following Deferred is getting paused: Deferred@1483981586(state=PAUSED, result=Deferred@493965016, callback=org.apache.kudu.client.AsyncKuduClient$6@60909d1 -> wakeup thread pool-3-thread-26, errback=passthrough -> wakeup thread pool-3-thread-26) 01:26:48.983 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:48.986 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:48.986 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:48.987 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:48.987 [DEBUG - New I/O worker #753] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:48.987 [DEBUG - New I/O worker #753] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:48.988 [DEBUG - pool-3-thread-26] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:48.989 [DEBUG - New I/O worker #815] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:48.988 [DEBUG - New I/O worker #808] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:48.989 [DEBUG - New I/O worker #753] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:48.990 [DEBUG - New I/O worker #755] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.991 [DEBUG - New I/O worker #819] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:48.990 [DEBUG - New I/O worker #865] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.997 [DEBUG - New I/O worker #770] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:48.998 [DEBUG - New I/O worker #828] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:48.999 [DEBUG - New I/O worker #795] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.004 [DEBUG - New I/O worker #755] (Negotiator.java:764) Authenticated connection [id: 0x49056b43, /127.1.193.125:50360 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.007 [DEBUG - New I/O worker #770] (Negotiator.java:764) Authenticated connection [id: 0xe0bdbb04, /127.1.193.125:50363 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.008 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.008 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=106), Traces: [0ms] refreshing cache from master, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [105ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.009 [DEBUG - New I/O worker #755] (Deferred.java:1330) callback=retry RPC@289509596 returned Deferred@360580652(state=PENDING, result=null, callback=(continuation of Deferred@1350343399 after retry RPC@289509596), errback=(continuation of Deferred@1350343399 after retry RPC@289509596)), so the following Deferred is getting paused: Deferred@1350343399(state=PAUSED, result=Deferred@360580652, callback=org.apache.kudu.client.AsyncKuduClient$6@7cbce139 -> wakeup thread pool-3-thread-12, errback=passthrough -> wakeup thread pool-3-thread-12) 01:26:49.012 [DEBUG - New I/O worker #795] (Negotiator.java:764) Authenticated connection [id: 0x013cb176, /127.1.193.124:46288 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.013 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.013 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=112), Traces: [0ms] refreshing cache from master, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [112ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.014 [DEBUG - New I/O worker #828] (Negotiator.java:764) Authenticated connection [id: 0x3c078edf, /127.1.193.126:51102 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.014 [DEBUG - New I/O worker #770] (Deferred.java:1330) callback=retry RPC@1631072919 returned Deferred@1695001192(state=PENDING, result=null, callback=(continuation of Deferred@537213100 after retry RPC@1631072919), errback=(continuation of Deferred@537213100 after retry RPC@1631072919)), so the following Deferred is getting paused: Deferred@537213100(state=PAUSED, result=Deferred@1695001192, callback=org.apache.kudu.client.AsyncKuduClient$6@3163dab5 -> wakeup thread pool-3-thread-17, errback=passthrough -> wakeup thread pool-3-thread-17) 01:26:49.016 [DEBUG - New I/O worker #819] (Negotiator.java:764) Authenticated connection [id: 0x7b0416d1, /127.1.193.124:46285 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.018 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.021 [DEBUG - New I/O worker #745] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.023 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.018 [DEBUG - New I/O worker #865] (Negotiator.java:764) Authenticated connection [id: 0x8be3a625, /127.1.193.126:51099 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.174 [DEBUG - New I/O worker #744] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.173 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.179 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.180 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.180 [DEBUG - New I/O worker #755] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.181 [DEBUG - New I/O worker #755] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.172 [DEBUG - pool-3-thread-53] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.182 [DEBUG - pool-3-thread-12] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.171 [DEBUG - pool-3-thread-50] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.170 [DEBUG - pool-3-thread-19] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.156 [DEBUG - New I/O worker #864] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.184 [DEBUG - New I/O worker #819] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.185 [DEBUG - New I/O worker #755] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.154 [DEBUG - New I/O worker #749] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.154 [DEBUG - New I/O worker #817] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.189 [DEBUG - New I/O worker #745] (Negotiator.java:764) Authenticated connection [id: 0x3a401467, /127.1.193.124:46291 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.190 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.190 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.191 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.191 [DEBUG - New I/O worker #770] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.192 [DEBUG - New I/O worker #770] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.021 [DEBUG - New I/O worker #809] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.195 [DEBUG - pool-3-thread-17] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.195 [DEBUG - New I/O worker #795] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.184 [DEBUG - New I/O worker #865] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.198 [DEBUG - New I/O worker #770] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.196 [DEBUG - New I/O worker #828] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.202 [DEBUG - New I/O worker #864] (Negotiator.java:764) Authenticated connection [id: 0x4a9ae3a3, /127.1.193.126:51108 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.202 [DEBUG - New I/O worker #749] (Negotiator.java:764) Authenticated connection [id: 0x31c088fd, /127.1.193.125:50369 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.206 [DEBUG - New I/O worker #744] (Negotiator.java:764) Authenticated connection [id: 0xd2b4571c, /127.1.193.125:50366 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.207 [DEBUG - New I/O worker #817] (Negotiator.java:764) Authenticated connection [id: 0xa224d8a2, /127.1.193.124:46294 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.211 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.212 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=311), Traces: [0ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [293ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.213 [DEBUG - New I/O worker #809] (Negotiator.java:764) Authenticated connection [id: 0x1d5028e7, /127.1.193.126:51105 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.214 [DEBUG - New I/O worker #744] (Deferred.java:1330) callback=retry RPC@155112891 returned Deferred@218185796(state=PENDING, result=null, callback=(continuation of Deferred@1208213376 after retry RPC@155112891), errback=(continuation of Deferred@1208213376 after retry RPC@155112891)), so the following Deferred is getting paused: Deferred@1208213376(state=PAUSED, result=Deferred@218185796, callback=org.apache.kudu.client.AsyncKuduClient$6@66ded3f -> wakeup thread pool-3-thread-7, errback=passthrough -> wakeup thread pool-3-thread-7) 01:26:49.216 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.216 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.219 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=266), Traces: [0ms] refreshing cache from master, [197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [259ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [262ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [264ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.220 [DEBUG - New I/O worker #749] (Deferred.java:1330) callback=retry RPC@595231848 returned Deferred@658834655(state=PENDING, result=null, callback=(continuation of Deferred@1648856659 after retry RPC@595231848), errback=(continuation of Deferred@1648856659 after retry RPC@595231848)), so the following Deferred is getting paused: Deferred@1648856659(state=PAUSED, result=Deferred@658834655, callback=org.apache.kudu.client.AsyncKuduClient$6@24c155a3 -> wakeup thread pool-3-thread-23, errback=passthrough -> wakeup thread pool-3-thread-23) 01:26:49.222 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.222 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.222 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.223 [DEBUG - New I/O worker #744] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.223 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.223 [DEBUG - New I/O worker #744] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.228 [DEBUG - pool-3-thread-35] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.229 [DEBUG - New I/O worker #779] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.226 [DEBUG - pool-3-thread-34] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.232 [DEBUG - New I/O worker #830] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.232 [DEBUG - pool-3-thread-7] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.233 [DEBUG - New I/O worker #744] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.234 [DEBUG - New I/O worker #745] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.234 [DEBUG - New I/O worker #776] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.235 [DEBUG - New I/O worker #809] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.235 [DEBUG - New I/O worker #792] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.236 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.230 [DEBUG - New I/O worker #793] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.235 [DEBUG - New I/O worker #849] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.234 [DEBUG - New I/O worker #805] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.241 [DEBUG - New I/O worker #762] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.241 [DEBUG - New I/O worker #777] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.246 [DEBUG - New I/O worker #779] (Negotiator.java:764) Authenticated connection [id: 0x8bc0fb93, /127.1.193.124:46297 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.250 [DEBUG - New I/O worker #830] (Negotiator.java:764) Authenticated connection [id: 0x6d378e30, /127.1.193.126:51114 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.250 [DEBUG - New I/O worker #792] (Negotiator.java:764) Authenticated connection [id: 0x197cf859, /127.1.193.126:51111 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.255 [DEBUG - New I/O worker #793] (Negotiator.java:764) Authenticated connection [id: 0x4a6a18eb, /127.1.193.124:46300 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.265 [DEBUG - New I/O worker #777] (Negotiator.java:764) Authenticated connection [id: 0xf3472106, /127.1.193.125:50372 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.267 [DEBUG - New I/O worker #849] (Negotiator.java:764) Authenticated connection [id: 0xd0203678, /127.1.193.126:51117 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.265 [DEBUG - New I/O worker #776] (Negotiator.java:764) Authenticated connection [id: 0xba7edd49, /127.1.193.125:50375 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.272 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.272 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=388), Traces: [0ms] refreshing cache from master, [332ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [332ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [333ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [370ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [371ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [387ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.273 [DEBUG - New I/O worker #777] (Deferred.java:1330) callback=retry RPC@2060555010 returned Deferred@2129533056(state=PENDING, result=null, callback=(continuation of Deferred@1005361465 after retry RPC@2060555010), errback=(continuation of Deferred@1005361465 after retry RPC@2060555010)), so the following Deferred is getting paused: Deferred@1005361465(state=PAUSED, result=Deferred@2129533056, callback=org.apache.kudu.client.AsyncKuduClient$6@7c22e447 -> wakeup thread pool-3-thread-25, errback=passthrough -> wakeup thread pool-3-thread-25) 01:26:49.274 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.277 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=95), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [77ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [79ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.276 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.278 [DEBUG - New I/O worker #762] (Negotiator.java:764) Authenticated connection [id: 0x321ef34f, /127.1.193.125:50378 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.277 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.278 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.279 [DEBUG - New I/O worker #749] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.280 [DEBUG - New I/O worker #749] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.281 [DEBUG - pool-3-thread-23] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.282 [DEBUG - New I/O worker #817] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.282 [DEBUG - New I/O worker #864] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.283 [DEBUG - New I/O worker #749] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.284 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.284 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=394), Traces: [0ms] refreshing cache from master, [337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [384ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [393ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.285 [DEBUG - New I/O worker #762] (Deferred.java:1330) callback=retry RPC@414664907 returned Deferred@478732037(state=PENDING, result=null, callback=(continuation of Deferred@1502233328 after retry RPC@414664907), errback=(continuation of Deferred@1502233328 after retry RPC@414664907)), so the following Deferred is getting paused: Deferred@1502233328(state=PAUSED, result=Deferred@478732037, callback=org.apache.kudu.client.AsyncKuduClient$6@4aca96a8 -> wakeup thread pool-3-thread-39, errback=passthrough -> wakeup thread pool-3-thread-39) 01:26:49.286 [DEBUG - New I/O worker #776] (Deferred.java:1330) callback=retry RPC@184163753 returned Deferred@247830574(state=PENDING, result=null, callback=(continuation of Deferred@1271338898 after retry RPC@184163753), errback=(continuation of Deferred@1271338898 after retry RPC@184163753)), so the following Deferred is getting paused: Deferred@1271338898(state=PAUSED, result=Deferred@247830574, callback=org.apache.kudu.client.AsyncKuduClient$6@200bbab7 -> wakeup thread pool-3-thread-53, errback=passthrough -> wakeup thread pool-3-thread-53) 01:26:49.294 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.292 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.298 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.287 [DEBUG - New I/O worker #805] (Negotiator.java:764) Authenticated connection [id: 0x7a66cf7c, /127.1.193.124:46303 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.299 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.300 [DEBUG - New I/O worker #777] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.300 [DEBUG - New I/O worker #777] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.301 [DEBUG - pool-3-thread-25] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.302 [DEBUG - New I/O worker #779] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.302 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.303 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.303 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.303 [DEBUG - New I/O worker #762] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.304 [DEBUG - New I/O worker #762] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.305 [DEBUG - pool-3-thread-39] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.305 [DEBUG - New I/O worker #762] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.305 [DEBUG - New I/O worker #849] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.297 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.305 [DEBUG - New I/O worker #805] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.303 [DEBUG - New I/O worker #777] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.302 [DEBUG - New I/O worker #792] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.315 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.316 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.317 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.317 [DEBUG - New I/O worker #776] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.318 [DEBUG - New I/O worker #776] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.319 [DEBUG - pool-3-thread-53] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.319 [DEBUG - New I/O worker #830] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.319 [DEBUG - New I/O worker #793] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.319 [DEBUG - New I/O worker #776] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.423 [DEBUG - pool-3-thread-73] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.427 [DEBUG - pool-3-thread-48] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.450 [DEBUG - pool-3-thread-71] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.455 [DEBUG - New I/O worker #1057] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.455 [DEBUG - New I/O worker #1062] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.455 [DEBUG - New I/O worker #1058] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.464 [DEBUG - New I/O worker #1057] (Negotiator.java:764) Authenticated connection [id: 0x058e20fd, /127.1.193.125:50381 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.465 [DEBUG - New I/O worker #1062] (Negotiator.java:764) Authenticated connection [id: 0x54de8571, /127.1.193.126:51120 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.467 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.468 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=16), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [16ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.468 [DEBUG - New I/O worker #1057] (Deferred.java:1330) callback=retry RPC@1675771080 returned Deferred@1742578796(state=PENDING, result=null, callback=(continuation of Deferred@585053939 after retry RPC@1675771080), errback=(continuation of Deferred@585053939 after retry RPC@1675771080)), so the following Deferred is getting paused: Deferred@585053939(state=PAUSED, result=Deferred@1742578796, callback=org.apache.kudu.client.AsyncKuduClient$6@61c1509b -> wakeup thread pool-3-thread-71, errback=passthrough -> wakeup thread pool-3-thread-71) 01:26:49.471 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.472 [DEBUG - New I/O worker #820] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.473 [DEBUG - New I/O worker #760] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.472 [DEBUG - New I/O worker #806] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.477 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.477 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.477 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.478 [DEBUG - New I/O worker #1057] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.478 [DEBUG - New I/O worker #1057] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.479 [DEBUG - pool-3-thread-71] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.480 [DEBUG - New I/O worker #1057] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.482 [DEBUG - New I/O worker #1062] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.472 [DEBUG - New I/O worker #1058] (Negotiator.java:764) Authenticated connection [id: 0x49f169ce, /127.1.193.124:46306 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.482 [DEBUG - New I/O worker #760] (Negotiator.java:764) Authenticated connection [id: 0x2454b0c4, /127.1.193.125:50384 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.483 [INFO - New I/O worker #1058] (Connection.java:435) [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:49.483 [DEBUG - New I/O worker #1058] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:49.485 [DEBUG - New I/O worker #806] (Negotiator.java:764) Authenticated connection [id: 0xb280aa77, /127.1.193.124:46309 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.485 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.487 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=256), Traces: [0ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [255ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.493 [DEBUG - New I/O worker #853] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.486 [DEBUG - New I/O worker #820] (Negotiator.java:764) Authenticated connection [id: 0x6e2fd277, /127.1.193.126:51123 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.493 [DEBUG - New I/O worker #800] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.494 [DEBUG - New I/O worker #767] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.497 [DEBUG - New I/O worker #760] (Deferred.java:1330) callback=retry RPC@1205715621 returned Deferred@1138904710(state=PENDING, result=null, callback=(continuation of Deferred@115394718 after retry RPC@1205715621), errback=(continuation of Deferred@115394718 after retry RPC@1205715621)), so the following Deferred is getting paused: Deferred@115394718(state=PAUSED, result=Deferred@1138904710, callback=org.apache.kudu.client.AsyncKuduClient$6@1ecd6821 -> wakeup thread pool-3-thread-34, errback=passthrough -> wakeup thread pool-3-thread-34) 01:26:49.499 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.508 [DEBUG - New I/O worker #853] (Negotiator.java:764) Authenticated connection [id: 0x7a1c75e4, /127.1.193.126:51126 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.509 [DEBUG - New I/O worker #800] (Negotiator.java:764) Authenticated connection [id: 0xb921c679, /127.1.193.124:46312 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.511 [DEBUG - New I/O worker #767] (Negotiator.java:764) Authenticated connection [id: 0xf1bc33c8, /127.1.193.125:50387 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.512 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.514 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.514 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.514 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.515 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=285), Traces: [0ms] refreshing cache from master, [260ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [282ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [283ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.516 [DEBUG - New I/O worker #767] (Deferred.java:1330) callback=retry RPC@1786013066 returned Deferred@1850473500(state=PENDING, result=null, callback=(continuation of Deferred@726233009 after retry RPC@1786013066), errback=(continuation of Deferred@726233009 after retry RPC@1786013066)), so the following Deferred is getting paused: Deferred@726233009(state=PAUSED, result=Deferred@1850473500, callback=org.apache.kudu.client.AsyncKuduClient$6@53b98543 -> wakeup thread pool-3-thread-35, errback=passthrough -> wakeup thread pool-3-thread-35) 01:26:49.516 [DEBUG - New I/O worker #760] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.517 [DEBUG - New I/O worker #760] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.520 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.526 [DEBUG - pool-3-thread-34] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.526 [DEBUG - New I/O worker #806] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.526 [DEBUG - New I/O worker #760] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.526 [DEBUG - New I/O worker #820] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.531 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.532 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.533 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.533 [DEBUG - New I/O worker #767] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.534 [DEBUG - New I/O worker #765] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.535 [DEBUG - New I/O worker #802] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.535 [DEBUG - New I/O worker #801] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.537 [DEBUG - New I/O worker #767] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.543 [DEBUG - New I/O worker #801] (Negotiator.java:764) Authenticated connection [id: 0xa5d72d00, /127.1.193.124:46315 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.545 [DEBUG - New I/O worker #767] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.546 [DEBUG - New I/O worker #800] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.545 [DEBUG - pool-3-thread-35] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.546 [DEBUG - New I/O worker #853] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.549 [DEBUG - New I/O worker #802] (Negotiator.java:764) Authenticated connection [id: 0x0e8e4775, /127.1.193.126:51129 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.553 [DEBUG - New I/O worker #765] (Negotiator.java:764) Authenticated connection [id: 0x307c9e26, /127.1.193.125:50390 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.555 [DEBUG - New I/O worker #1029] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.556 [DEBUG - New I/O worker #1031] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.557 [DEBUG - New I/O worker #1032] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.563 [DEBUG - New I/O worker #1029] (Negotiator.java:764) Authenticated connection [id: 0x8e79ef5f, /127.1.193.125:50393 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.563 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.564 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=379), Traces: [0ms] refreshing cache from master, [348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [348ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [364ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [369ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [379ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.566 [DEBUG - New I/O worker #1031] (Negotiator.java:764) Authenticated connection [id: 0x7b6fcbc1, /127.1.193.124:46318 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.566 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.566 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=383), Traces: [0ms] refreshing cache from master, [370ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [371ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [371ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [383ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.567 [DEBUG - New I/O worker #765] (Deferred.java:1330) callback=retry RPC@1197773083 returned Deferred@1130044719(state=PENDING, result=null, callback=(continuation of Deferred@106535712 after retry RPC@1197773083), errback=(continuation of Deferred@106535712 after retry RPC@1197773083)), so the following Deferred is getting paused: Deferred@106535712(state=PAUSED, result=Deferred@1130044719, callback=org.apache.kudu.client.AsyncKuduClient$6@549a043a -> wakeup thread pool-3-thread-19, errback=passthrough -> wakeup thread pool-3-thread-19) 01:26:49.569 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.567 [DEBUG - New I/O worker #1029] (Deferred.java:1330) callback=retry RPC@1141719099 returned Deferred@1077075042(state=PENDING, result=null, callback=(continuation of Deferred@87050752 after retry RPC@1141719099), errback=(continuation of Deferred@87050752 after retry RPC@1141719099)), so the following Deferred is getting paused: Deferred@87050752(state=PAUSED, result=Deferred@1077075042, callback=org.apache.kudu.client.AsyncKuduClient$6@1a5d0f93 -> wakeup thread pool-3-thread-50, errback=passthrough -> wakeup thread pool-3-thread-50) 01:26:49.571 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.570 [DEBUG - New I/O worker #1032] (Negotiator.java:764) Authenticated connection [id: 0x12f210e7, /127.1.193.126:51132 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.575 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.576 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.576 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.576 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.576 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.577 [DEBUG - New I/O worker #1029] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.577 [DEBUG - New I/O worker #1029] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.576 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.578 [DEBUG - New I/O worker #1031] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.578 [DEBUG - New I/O worker #765] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.578 [DEBUG - New I/O worker #1032] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.578 [DEBUG - pool-3-thread-50] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.579 [DEBUG - New I/O worker #1029] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.579 [DEBUG - New I/O worker #765] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.581 [DEBUG - pool-3-thread-19] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.581 [DEBUG - New I/O worker #801] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.582 [DEBUG - New I/O worker #802] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.582 [DEBUG - New I/O worker #765] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.595 [DEBUG - pool-3-thread-32] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.484 [INFO - New I/O worker #1058] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:49.748 [DEBUG - New I/O worker #1022] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.749 [DEBUG - New I/O worker #1182] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.754 [DEBUG - New I/O worker #1181] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.756 [DEBUG - New I/O worker #1014] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.756 [DEBUG - New I/O worker #1011] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.761 [DEBUG - New I/O worker #1017] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.768 [DEBUG - New I/O worker #1011] (Negotiator.java:764) Authenticated connection [id: 0xecb5a978, /127.1.193.125:50399 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.769 [DEBUG - New I/O worker #1181] (Negotiator.java:764) Authenticated connection [id: 0x1f276b25, /127.1.193.124:46321 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.771 [DEBUG - New I/O worker #1014] (Negotiator.java:764) Authenticated connection [id: 0x6762296c, /127.1.193.124:46324 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.775 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.775 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=351), Traces: [0ms] refreshing cache from master, [328ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [348ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [350ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:49.775 [DEBUG - New I/O worker #1182] (Negotiator.java:764) Authenticated connection [id: 0x13e89a47, /127.1.193.126:51135 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.776 [DEBUG - New I/O worker #1011] (Deferred.java:1330) callback=retry RPC@1720826242 returned Deferred@1655579016(state=PENDING, result=null, callback=(continuation of Deferred@665629625 after retry RPC@1720826242), errback=(continuation of Deferred@665629625 after retry RPC@1720826242)), so the following Deferred is getting paused: Deferred@665629625(state=PAUSED, result=Deferred@1655579016, callback=org.apache.kudu.client.AsyncKuduClient$6@7d772c3e -> wakeup thread pool-3-thread-73, errback=passthrough -> wakeup thread pool-3-thread-73) 01:26:49.779 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.781 [DEBUG - New I/O worker #1017] (Negotiator.java:764) Authenticated connection [id: 0x2d2597f2, /127.1.193.126:51138 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.782 [DEBUG - New I/O worker #1022] (Negotiator.java:764) Authenticated connection [id: 0xb2d28780, /127.1.193.125:50396 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.784 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.784 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.785 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.785 [DEBUG - New I/O worker #1011] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.785 [DEBUG - New I/O worker #1011] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.786 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.787 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=359), Traces: [0ms] refreshing cache from master, [320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [345ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [352ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [359ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.787 [DEBUG - New I/O worker #1022] (Deferred.java:1330) callback=retry RPC@725779538 returned Deferred@796726754(state=PENDING, result=null, callback=(continuation of Deferred@1786744425 after retry RPC@725779538), errback=(continuation of Deferred@1786744425 after retry RPC@725779538)), so the following Deferred is getting paused: Deferred@1786744425(state=PAUSED, result=Deferred@796726754, callback=org.apache.kudu.client.AsyncKuduClient$6@64000c5a -> wakeup thread pool-3-thread-48, errback=passthrough -> wakeup thread pool-3-thread-48) 01:26:49.788 [DEBUG - New I/O worker #1017] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.787 [DEBUG - pool-3-thread-73] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.788 [DEBUG - New I/O worker #1011] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.788 [DEBUG - New I/O worker #1014] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.791 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.795 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.796 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.796 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.796 [DEBUG - New I/O worker #1022] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.797 [DEBUG - New I/O worker #1022] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.799 [DEBUG - pool-3-thread-48] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.799 [DEBUG - New I/O worker #1182] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.799 [DEBUG - New I/O worker #1181] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.800 [DEBUG - New I/O worker #1022] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.836 [DEBUG - pool-3-thread-33] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.859 [DEBUG - pool-3-thread-61] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.866 [DEBUG - New I/O worker #1153] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.866 [DEBUG - New I/O worker #975] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.866 [DEBUG - New I/O worker #1161] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.873 [DEBUG - pool-3-thread-66] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:49.878 [DEBUG - New I/O worker #975] (Negotiator.java:764) Authenticated connection [id: 0x0fb77b63, /127.1.193.125:50402 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.880 [DEBUG - New I/O worker #1153] (Negotiator.java:764) Authenticated connection [id: 0x972458c7, /127.1.193.124:46327 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.881 [DEBUG - New I/O worker #1161] (Negotiator.java:764) Authenticated connection [id: 0xc59bc554, /127.1.193.126:51141 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.882 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.883 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=24), Traces: [0ms] refreshing cache from master, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [22ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.887 [DEBUG - New I/O worker #975] (Deferred.java:1330) callback=retry RPC@1200805955 returned Deferred@1135432809(state=PENDING, result=null, callback=(continuation of Deferred@112185976 after retry RPC@1200805955), errback=(continuation of Deferred@112185976 after retry RPC@1200805955)), so the following Deferred is getting paused: Deferred@112185976(state=PAUSED, result=Deferred@1135432809, callback=org.apache.kudu.client.AsyncKuduClient$6@66628843 -> wakeup thread pool-3-thread-61, errback=passthrough -> wakeup thread pool-3-thread-61) 01:26:49.887 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.892 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.892 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.892 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.893 [DEBUG - New I/O worker #975] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.893 [DEBUG - New I/O worker #975] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.894 [DEBUG - pool-3-thread-61] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.895 [DEBUG - New I/O worker #1161] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.895 [DEBUG - New I/O worker #975] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.895 [DEBUG - New I/O worker #1153] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.905 [DEBUG - New I/O worker #997] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:49.906 [DEBUG - New I/O worker #1134] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:49.907 [DEBUG - New I/O worker #1135] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:49.913 [DEBUG - New I/O worker #997] (Negotiator.java:764) Authenticated connection [id: 0x380ab98f, /127.1.193.125:50405 => /127.1.193.125:37435] using SASL/PLAIN 01:26:49.916 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:49.916 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=79), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [78ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:49.917 [DEBUG - New I/O worker #1135] (Negotiator.java:764) Authenticated connection [id: 0xc0da1c6d, /127.1.193.126:51144 => /127.1.193.126:40739] using SASL/PLAIN 01:26:49.917 [DEBUG - New I/O worker #997] (Deferred.java:1330) callback=retry RPC@1141152550 returned Deferred@1077614462(state=PENDING, result=null, callback=(continuation of Deferred@87658781 after retry RPC@1141152550), errback=(continuation of Deferred@87658781 after retry RPC@1141152550)), so the following Deferred is getting paused: Deferred@87658781(state=PAUSED, result=Deferred@1077614462, callback=org.apache.kudu.client.AsyncKuduClient$6@54cbbeb8 -> wakeup thread pool-3-thread-33, errback=passthrough -> wakeup thread pool-3-thread-33) 01:26:49.919 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:49.922 [DEBUG - New I/O worker #1134] (Negotiator.java:764) Authenticated connection [id: 0xb5bc2b7f, /127.1.193.124:46330 => /127.1.193.124:58247] using SASL/PLAIN 01:26:49.924 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:49.924 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:49.924 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:49.925 [DEBUG - New I/O worker #997] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:49.925 [DEBUG - New I/O worker #997] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:49.926 [DEBUG - pool-3-thread-33] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:49.926 [DEBUG - New I/O worker #1134] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:49.927 [DEBUG - New I/O worker #1135] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:49.927 [DEBUG - New I/O worker #997] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:49.992 [DEBUG - pool-3-thread-31] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.004 [DEBUG - pool-3-thread-72] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.016 [DEBUG - pool-3-thread-21] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.019 [DEBUG - New I/O worker #1087] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.020 [DEBUG - pool-3-thread-26] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.020 [DEBUG - New I/O worker #1088] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.020 [DEBUG - New I/O worker #1082] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.029 [DEBUG - New I/O worker #1087] (Negotiator.java:764) Authenticated connection [id: 0x478af5a2, /127.1.193.124:46333 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.032 [DEBUG - New I/O worker #1082] (Negotiator.java:764) Authenticated connection [id: 0x1d0b6e14, /127.1.193.125:50408 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.033 [DEBUG - New I/O worker #1088] (Negotiator.java:764) Authenticated connection [id: 0x8c518bfe, /127.1.193.126:51147 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.038 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.039 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=443), Traces: [0ms] refreshing cache from master, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [422ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [438ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [442ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.041 [DEBUG - New I/O worker #1092] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.042 [DEBUG - New I/O worker #1082] (Deferred.java:1330) callback=retry RPC@425383206 returned Deferred@493175509(state=PENDING, result=null, callback=(continuation of Deferred@1483202333 after retry RPC@425383206), errback=(continuation of Deferred@1483202333 after retry RPC@425383206)), so the following Deferred is getting paused: Deferred@1483202333(state=PAUSED, result=Deferred@493175509, callback=org.apache.kudu.client.AsyncKuduClient$6@264915b5 -> wakeup thread pool-3-thread-32, errback=passthrough -> wakeup thread pool-3-thread-32) 01:26:50.042 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.043 [DEBUG - New I/O worker #1080] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.045 [DEBUG - New I/O worker #1236] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.047 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.052 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.053 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.053 [DEBUG - New I/O worker #1082] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.053 [DEBUG - New I/O worker #1082] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.054 [DEBUG - New I/O worker #1087] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.055 [DEBUG - New I/O worker #1088] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.058 [DEBUG - New I/O worker #1078] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.059 [DEBUG - New I/O worker #1083] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.052 [DEBUG - New I/O worker #1092] (Negotiator.java:764) Authenticated connection [id: 0xe61a744b, /127.1.193.124:46336 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.055 [DEBUG - New I/O worker #1236] (Negotiator.java:764) Authenticated connection [id: 0x3f2eeb84, /127.1.193.126:51150 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.055 [DEBUG - New I/O worker #1082] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.054 [DEBUG - pool-3-thread-32] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.053 [DEBUG - New I/O worker #1080] (Negotiator.java:764) Authenticated connection [id: 0xb4187f9e, /127.1.193.125:50411 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.065 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.065 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=44), Traces: [0ms] refreshing cache from master, [16ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [17ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [17ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [44ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:50.066 [DEBUG - New I/O worker #1081] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.068 [DEBUG - New I/O worker #1080] (Deferred.java:1330) callback=retry RPC@55786295 returned Deferred@124563333(state=PENDING, result=null, callback=(continuation of Deferred@1114517772 after retry RPC@55786295), errback=(continuation of Deferred@1114517772 after retry RPC@55786295)), so the following Deferred is getting paused: Deferred@1114517772(state=PAUSED, result=Deferred@124563333, callback=org.apache.kudu.client.AsyncKuduClient$6@6976f6d8 -> wakeup thread pool-3-thread-26, errback=passthrough -> wakeup thread pool-3-thread-26) 01:26:50.069 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.073 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.073 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.073 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.073 [DEBUG - New I/O worker #1081] (Negotiator.java:764) Authenticated connection [id: 0x924cdd52, /127.1.193.124:46339 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.076 [DEBUG - New I/O worker #1083] (Negotiator.java:764) Authenticated connection [id: 0x7fa95b0f, /127.1.193.126:51153 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.073 [DEBUG - New I/O worker #1080] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.077 [DEBUG - New I/O worker #1078] (Negotiator.java:764) Authenticated connection [id: 0xc27435eb, /127.1.193.125:50414 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.080 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.081 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=64), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [59ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.082 [DEBUG - New I/O worker #1078] (Deferred.java:1330) callback=retry RPC@1083141887 returned Deferred@1152447030(state=PENDING, result=null, callback=(continuation of Deferred@28468420 after retry RPC@1083141887), errback=(continuation of Deferred@28468420 after retry RPC@1083141887)), so the following Deferred is getting paused: Deferred@28468420(state=PAUSED, result=Deferred@1152447030, callback=org.apache.kudu.client.AsyncKuduClient$6@b3a5f1d -> wakeup thread pool-3-thread-21, errback=passthrough -> wakeup thread pool-3-thread-21) 01:26:50.082 [DEBUG - New I/O worker #1080] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299995, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299995, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.083 [DEBUG - pool-3-thread-26] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.084 [DEBUG - New I/O worker #1236] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.084 [DEBUG - New I/O worker #1092] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.085 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.086 [DEBUG - New I/O worker #1080] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.089 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.089 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.089 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.090 [DEBUG - New I/O worker #1078] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.090 [DEBUG - New I/O worker #1078] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.091 [DEBUG - pool-3-thread-21] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.091 [DEBUG - New I/O worker #1083] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.091 [DEBUG - New I/O worker #1081] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.091 [DEBUG - New I/O worker #1078] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.106 [DEBUG - New I/O worker #1104] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.107 [DEBUG - New I/O worker #1097] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.109 [DEBUG - New I/O worker #1066] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.110 [DEBUG - New I/O worker #1074] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.111 [DEBUG - New I/O worker #1212] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.111 [DEBUG - New I/O worker #1098] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.122 [DEBUG - New I/O worker #1212] (Negotiator.java:764) Authenticated connection [id: 0x03f989c6, /127.1.193.126:51158 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.122 [DEBUG - New I/O worker #1104] (Negotiator.java:764) Authenticated connection [id: 0xec4c0b80, /127.1.193.124:46343 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.122 [DEBUG - New I/O worker #1097] (Negotiator.java:764) Authenticated connection [id: 0xc9269dfd, /127.1.193.124:46344 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.122 [DEBUG - New I/O worker #1074] (Negotiator.java:764) Authenticated connection [id: 0x835dc5e2, /127.1.193.125:50418 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.125 [DEBUG - New I/O worker #1098] (Negotiator.java:764) Authenticated connection [id: 0x3d00c291, /127.1.193.126:51159 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.126 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.126 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=133), Traces: [0ms] refreshing cache from master, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [132ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.128 [DEBUG - New I/O worker #1066] (Negotiator.java:764) Authenticated connection [id: 0x065243ec, /127.1.193.125:50417 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.129 [DEBUG - New I/O worker #1074] (Deferred.java:1330) callback=retry RPC@1662325893 returned Deferred@1730841663(state=PENDING, result=null, callback=(continuation of Deferred@573054654 after retry RPC@1662325893), errback=(continuation of Deferred@573054654 after retry RPC@1662325893)), so the following Deferred is getting paused: Deferred@573054654(state=PAUSED, result=Deferred@1730841663, callback=org.apache.kudu.client.AsyncKuduClient$6@583a7d2b -> wakeup thread pool-3-thread-31, errback=passthrough -> wakeup thread pool-3-thread-31) 01:26:50.129 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.131 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.132 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=127), Traces: [0ms] refreshing cache from master, [98ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [121ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [127ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.135 [DEBUG - New I/O worker #1066] (Deferred.java:1330) callback=retry RPC@1231371955 returned Deferred@1297789767(state=PENDING, result=null, callback=(continuation of Deferred@139998344 after retry RPC@1231371955), errback=(continuation of Deferred@139998344 after retry RPC@1231371955)), so the following Deferred is getting paused: Deferred@139998344(state=PAUSED, result=Deferred@1297789767, callback=org.apache.kudu.client.AsyncKuduClient$6@59ea36d4 -> wakeup thread pool-3-thread-72, errback=passthrough -> wakeup thread pool-3-thread-72) 01:26:50.139 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.141 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.141 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.141 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.142 [DEBUG - New I/O worker #1074] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.142 [DEBUG - New I/O worker #1074] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.143 [DEBUG - New I/O worker #1074] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.143 [DEBUG - New I/O worker #1098] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.143 [DEBUG - pool-3-thread-31] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.144 [DEBUG - New I/O worker #1097] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.146 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.146 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.146 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.146 [DEBUG - New I/O worker #1066] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.147 [DEBUG - New I/O worker #1066] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.147 [DEBUG - pool-3-thread-72] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.148 [DEBUG - New I/O worker #1212] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.148 [DEBUG - New I/O worker #1104] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.148 [DEBUG - New I/O worker #1066] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.156 [DEBUG - pool-3-thread-37] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.157 [DEBUG - pool-3-thread-74] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.158 [DEBUG - pool-3-thread-8] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.168 [DEBUG - pool-3-thread-68] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.171 [DEBUG - pool-3-thread-27] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.180 [DEBUG - pool-3-thread-30] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.182 [DEBUG - New I/O worker #981] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.183 [DEBUG - New I/O worker #1165] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.183 [DEBUG - New I/O worker #1166] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.193 [DEBUG - New I/O worker #1165] (Negotiator.java:764) Authenticated connection [id: 0xd75bd339, /127.1.193.124:46348 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.194 [DEBUG - New I/O worker #1166] (Negotiator.java:764) Authenticated connection [id: 0x08dba182, /127.1.193.126:51162 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.195 [DEBUG - New I/O worker #981] (Negotiator.java:764) Authenticated connection [id: 0x8b44ceb2, /127.1.193.125:50423 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.199 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.200 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=326), Traces: [0ms] refreshing cache from master, [306ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [306ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [307ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [323ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [324ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [326ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.203 [DEBUG - New I/O worker #981] (Deferred.java:1330) callback=retry RPC@1331465919 returned Deferred@1264780172(state=PENDING, result=null, callback=(continuation of Deferred@241271940 after retry RPC@1331465919), errback=(continuation of Deferred@241271940 after retry RPC@1331465919)), so the following Deferred is getting paused: Deferred@241271940(state=PAUSED, result=Deferred@1264780172, callback=org.apache.kudu.client.AsyncKuduClient$6@14a7d13c -> wakeup thread pool-3-thread-66, errback=passthrough -> wakeup thread pool-3-thread-66) 01:26:50.204 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.206 [DEBUG - pool-3-thread-36] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.211 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.211 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.212 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.212 [DEBUG - New I/O worker #981] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.214 [DEBUG - New I/O worker #981] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299997, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299997, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.215 [DEBUG - pool-3-thread-66] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.216 [DEBUG - New I/O worker #1165] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.216 [DEBUG - New I/O worker #1166] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.216 [DEBUG - New I/O worker #981] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.226 [DEBUG - pool-3-thread-49] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.228 [DEBUG - pool-3-thread-45] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.232 [DEBUG - New I/O worker #973] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.235 [DEBUG - New I/O worker #1154] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.239 [DEBUG - New I/O worker #971] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.241 [DEBUG - New I/O worker #973] (Negotiator.java:764) Authenticated connection [id: 0xda2c6c2e, /127.1.193.124:46351 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.243 [DEBUG - New I/O worker #1154] (Negotiator.java:764) Authenticated connection [id: 0xbce3d980, /127.1.193.126:51165 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.244 [DEBUG - pool-3-thread-70] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.246 [DEBUG - New I/O worker #971] (Negotiator.java:764) Authenticated connection [id: 0x387e725e, /127.1.193.125:50426 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.249 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.249 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=42), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [37ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [39ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.250 [DEBUG - pool-3-thread-40] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.250 [DEBUG - New I/O worker #971] (Deferred.java:1330) callback=retry RPC@1998281567 returned Deferred@1931788644(state=PENDING, result=null, callback=(continuation of Deferred@908480868 after retry RPC@1998281567), errback=(continuation of Deferred@908480868 after retry RPC@1998281567)), so the following Deferred is getting paused: Deferred@908480868(state=PAUSED, result=Deferred@1931788644, callback=org.apache.kudu.client.AsyncKuduClient$6@2a36c1be -> wakeup thread pool-3-thread-36, errback=passthrough -> wakeup thread pool-3-thread-36) 01:26:50.253 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.253 [DEBUG - New I/O worker #1167] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.256 [DEBUG - New I/O worker #989] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.257 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.257 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.258 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.258 [DEBUG - New I/O worker #971] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.258 [DEBUG - New I/O worker #971] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.259 [DEBUG - pool-3-thread-36] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.260 [DEBUG - New I/O worker #973] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.260 [DEBUG - New I/O worker #971] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.261 [DEBUG - New I/O worker #1168] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.261 [DEBUG - New I/O worker #1154] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.264 [DEBUG - New I/O worker #1167] (Negotiator.java:764) Authenticated connection [id: 0x6ed42ed4, /127.1.193.124:46354 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.264 [DEBUG - New I/O worker #1171] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.265 [DEBUG - New I/O worker #1142] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.265 [DEBUG - New I/O worker #998] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.273 [DEBUG - New I/O worker #989] (Negotiator.java:764) Authenticated connection [id: 0xdbaa3ed3, /127.1.193.125:50429 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.274 [DEBUG - New I/O worker #1168] (Negotiator.java:764) Authenticated connection [id: 0x9699cc98, /127.1.193.126:51168 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.275 [DEBUG - New I/O worker #1142] (Negotiator.java:764) Authenticated connection [id: 0x88aa2e09, /127.1.193.124:46357 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.275 [DEBUG - New I/O worker #998] (Negotiator.java:764) Authenticated connection [id: 0xec02d091, /127.1.193.125:50432 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.276 [DEBUG - New I/O worker #1171] (Negotiator.java:764) Authenticated connection [id: 0x4e38306d, /127.1.193.126:51171 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.279 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.280 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=99), Traces: [0ms] refreshing cache from master, [71ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [71ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [71ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [88ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [98ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [99ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.280 [DEBUG - New I/O worker #989] (Deferred.java:1330) callback=retry RPC@625271839 returned Deferred@561738863(state=PENDING, result=null, callback=(continuation of Deferred@1685712420 after retry RPC@625271839), errback=(continuation of Deferred@1685712420 after retry RPC@625271839)), so the following Deferred is getting paused: Deferred@1685712420(state=PAUSED, result=Deferred@561738863, callback=org.apache.kudu.client.AsyncKuduClient$6@57316e53 -> wakeup thread pool-3-thread-30, errback=passthrough -> wakeup thread pool-3-thread-30) 01:26:50.281 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.281 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=110), Traces: [0ms] refreshing cache from master, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [106ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [108ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [110ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.284 [DEBUG - New I/O worker #998] (Deferred.java:1330) callback=retry RPC@1601209751 returned Deferred@1531910817(state=PENDING, result=null, callback=(continuation of Deferred@508399532 after retry RPC@1601209751), errback=(continuation of Deferred@508399532 after retry RPC@1601209751)), so the following Deferred is getting paused: Deferred@508399532(state=PAUSED, result=Deferred@1531910817, callback=org.apache.kudu.client.AsyncKuduClient$6@749e5c4d -> wakeup thread pool-3-thread-27, errback=passthrough -> wakeup thread pool-3-thread-27) 01:26:50.284 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.284 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.288 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.289 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.289 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.289 [DEBUG - pool-3-thread-2] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.290 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.291 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.291 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.291 [DEBUG - New I/O worker #998] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.292 [DEBUG - New I/O worker #998] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.293 [DEBUG - pool-3-thread-27] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.293 [DEBUG - New I/O worker #1142] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.294 [DEBUG - New I/O worker #998] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.294 [DEBUG - New I/O worker #1004] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.289 [DEBUG - New I/O worker #989] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.293 [DEBUG - New I/O worker #1171] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.296 [DEBUG - New I/O worker #1009] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.298 [DEBUG - New I/O worker #1007] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.304 [DEBUG - New I/O worker #1009] (Negotiator.java:764) Authenticated connection [id: 0x86850952, /127.1.193.126:51174 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.305 [DEBUG - New I/O worker #1004] (Negotiator.java:764) Authenticated connection [id: 0xde39b3e1, /127.1.193.125:50435 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.306 [DEBUG - New I/O worker #1007] (Negotiator.java:764) Authenticated connection [id: 0x1aefca8a, /127.1.193.124:46360 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.310 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.310 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=141), Traces: [0ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [138ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [140ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [140ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:50.311 [DEBUG - New I/O worker #989] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299986, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299986, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299986, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299986, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.312 [DEBUG - New I/O worker #1004] (Deferred.java:1330) callback=retry RPC@275756643 returned Deferred@340795574(state=PENDING, result=null, callback=(continuation of Deferred@1364376664 after retry RPC@275756643), errback=(continuation of Deferred@1364376664 after retry RPC@275756643)), so the following Deferred is getting paused: Deferred@1364376664(state=PAUSED, result=Deferred@340795574, callback=org.apache.kudu.client.AsyncKuduClient$6@df82bcf -> wakeup thread pool-3-thread-68, errback=passthrough -> wakeup thread pool-3-thread-68) 01:26:50.313 [DEBUG - pool-3-thread-30] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.314 [DEBUG - New I/O worker #1168] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.314 [DEBUG - New I/O worker #1167] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.315 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.316 [DEBUG - New I/O worker #989] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.320 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.320 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.320 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.320 [DEBUG - New I/O worker #1004] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.321 [DEBUG - New I/O worker #1004] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.322 [DEBUG - New I/O worker #1007] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.322 [DEBUG - pool-3-thread-68] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.322 [DEBUG - New I/O worker #1009] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.322 [DEBUG - New I/O worker #1004] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.340 [DEBUG - pool-3-thread-77] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.352 [DEBUG - New I/O worker #994] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.353 [DEBUG - New I/O worker #1003] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.354 [DEBUG - New I/O worker #1108] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.354 [DEBUG - New I/O worker #1001] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.356 [DEBUG - New I/O worker #1053] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.357 [DEBUG - New I/O worker #1107] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.387 [DEBUG - New I/O worker #1178] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.388 [DEBUG - New I/O worker #1012] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.388 [DEBUG - New I/O worker #994] (Negotiator.java:764) Authenticated connection [id: 0x5c45d835, /127.1.193.125:50440 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.389 [DEBUG - New I/O worker #1053] (Negotiator.java:764) Authenticated connection [id: 0x1f853c16, /127.1.193.125:50438 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.392 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.393 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=234), Traces: [0ms] refreshing cache from master, [189ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [189ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [189ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [233ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.393 [DEBUG - New I/O worker #1107] (Negotiator.java:764) Authenticated connection [id: 0x4a3a20a6, /127.1.193.124:46363 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.462 [DEBUG - pool-3-thread-18] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.461 [DEBUG - pool-3-thread-75] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.451 [DEBUG - pool-3-thread-69] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.442 [DEBUG - pool-3-thread-60] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.430 [DEBUG - pool-3-thread-62] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.409 [DEBUG - pool-3-thread-16] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.404 [DEBUG - New I/O worker #1279] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.401 [DEBUG - New I/O worker #1178] (Negotiator.java:764) Authenticated connection [id: 0x22f161dd, /127.1.193.124:46369 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.400 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.476 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=317), Traces: [0ms] refreshing cache from master, [190ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [243ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.476 [DEBUG - New I/O worker #1053] (Deferred.java:1330) callback=retry RPC@376595593 returned Deferred@307094175(state=PENDING, result=null, callback=(continuation of Deferred@1464823474 after retry RPC@376595593), errback=(continuation of Deferred@1464823474 after retry RPC@376595593)), so the following Deferred is getting paused: Deferred@1464823474(state=PAUSED, result=Deferred@307094175, callback=org.apache.kudu.client.AsyncKuduClient$6@63816f45 -> wakeup thread pool-3-thread-74, errback=passthrough -> wakeup thread pool-3-thread-74) 01:26:50.398 [DEBUG - New I/O worker #1012] (Negotiator.java:764) Authenticated connection [id: 0xdeb97025, /127.1.193.125:50444 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.396 [DEBUG - New I/O worker #1001] (Negotiator.java:764) Authenticated connection [id: 0xeec20e84, /127.1.193.124:46366 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.395 [DEBUG - New I/O worker #1108] (Negotiator.java:764) Authenticated connection [id: 0x7200c54c, /127.1.193.126:51178 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.394 [DEBUG - New I/O worker #1003] (Negotiator.java:764) Authenticated connection [id: 0xacc6016f, /127.1.193.126:51180 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.393 [DEBUG - New I/O worker #994] (Deferred.java:1330) callback=retry RPC@734226340 returned Deferred@805103108(state=PENDING, result=null, callback=(continuation of Deferred@1795059103 after retry RPC@734226340), errback=(continuation of Deferred@1795059103 after retry RPC@734226340)), so the following Deferred is getting paused: Deferred@1795059103(state=PAUSED, result=Deferred@805103108, callback=org.apache.kudu.client.AsyncKuduClient$6@6b102299 -> wakeup thread pool-3-thread-8, errback=passthrough -> wakeup thread pool-3-thread-8) 01:26:50.479 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.480 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.473 [DEBUG - pool-3-thread-65] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.484 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.484 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.484 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.484 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.485 [DEBUG - New I/O worker #1053] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.485 [DEBUG - New I/O worker #1053] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.486 [DEBUG - New I/O worker #1107] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.487 [DEBUG - New I/O worker #1053] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.484 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=328), Traces: [0ms] refreshing cache from master, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [327ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [327ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.487 [DEBUG - New I/O worker #1108] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.486 [DEBUG - pool-3-thread-74] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.484 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.489 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.489 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.490 [DEBUG - New I/O worker #994] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.490 [DEBUG - New I/O worker #994] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.490 [DEBUG - pool-3-thread-47] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.491 [DEBUG - pool-3-thread-8] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.491 [DEBUG - New I/O worker #1003] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.491 [DEBUG - New I/O worker #1001] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.489 [DEBUG - New I/O worker #1012] (Deferred.java:1330) callback=retry RPC@797879499 returned Deferred@733036002(state=PENDING, result=null, callback=(continuation of Deferred@1857267440 after retry RPC@797879499), errback=(continuation of Deferred@1857267440 after retry RPC@797879499)), so the following Deferred is getting paused: Deferred@1857267440(state=PAUSED, result=Deferred@733036002, callback=org.apache.kudu.client.AsyncKuduClient$6@5258072 -> wakeup thread pool-3-thread-37, errback=passthrough -> wakeup thread pool-3-thread-37) 01:26:50.492 [DEBUG - New I/O worker #994] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.496 [DEBUG - New I/O worker #1279] (Negotiator.java:764) Authenticated connection [id: 0xab863601, /127.1.193.126:51183 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.496 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.501 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.501 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.502 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.502 [DEBUG - New I/O worker #1012] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.503 [DEBUG - New I/O worker #1012] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.503 [DEBUG - New I/O worker #1178] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.503 [DEBUG - pool-3-thread-37] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.505 [DEBUG - New I/O worker #1279] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.505 [DEBUG - New I/O worker #1012] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.508 [DEBUG - New I/O worker #1110] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.510 [DEBUG - New I/O worker #1052] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.510 [DEBUG - New I/O worker #1202] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.517 [DEBUG - New I/O worker #1110] (Negotiator.java:764) Authenticated connection [id: 0xcd637142, /127.1.193.124:46372 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.518 [DEBUG - New I/O worker #1052] (Negotiator.java:764) Authenticated connection [id: 0x1372af9c, /127.1.193.125:50447 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.520 [DEBUG - New I/O worker #1202] (Negotiator.java:764) Authenticated connection [id: 0x84eb9a41, /127.1.193.126:51186 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.527 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.527 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=186), Traces: [0ms] refreshing cache from master, [165ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [180ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [186ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.528 [DEBUG - New I/O worker #1052] (Deferred.java:1330) callback=retry RPC@556415417 returned Deferred@622180676(state=PENDING, result=null, callback=(continuation of Deferred@1612133250 after retry RPC@556415417), errback=(continuation of Deferred@1612133250 after retry RPC@556415417)), so the following Deferred is getting paused: Deferred@1612133250(state=PAUSED, result=Deferred@622180676, callback=org.apache.kudu.client.AsyncKuduClient$6@21aa31c6 -> wakeup thread pool-3-thread-77, errback=passthrough -> wakeup thread pool-3-thread-77) 01:26:50.531 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.535 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.535 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.536 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.536 [DEBUG - New I/O worker #1052] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.536 [DEBUG - New I/O worker #1052] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.537 [DEBUG - pool-3-thread-77] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.538 [DEBUG - New I/O worker #1202] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.538 [DEBUG - New I/O worker #1110] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.538 [DEBUG - New I/O worker #1052] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.543 [DEBUG - pool-3-thread-80] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.545 [DEBUG - New I/O worker #993] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.547 [DEBUG - New I/O worker #1169] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.548 [DEBUG - New I/O worker #1144] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.556 [DEBUG - New I/O worker #1169] (Negotiator.java:764) Authenticated connection [id: 0x89df6b23, /127.1.193.126:51189 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.556 [DEBUG - New I/O worker #993] (Negotiator.java:764) Authenticated connection [id: 0xd0b0c326, /127.1.193.125:50450 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.558 [DEBUG - New I/O worker #1144] (Negotiator.java:764) Authenticated connection [id: 0x38675736, /127.1.193.124:46375 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.560 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.561 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=271), Traces: [0ms] refreshing cache from master, [250ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [250ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [250ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [269ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [270ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:50.564 [DEBUG - New I/O worker #1147] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.564 [DEBUG - New I/O worker #1164] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.565 [DEBUG - New I/O worker #979] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.567 [DEBUG - New I/O worker #993] (Deferred.java:1330) callback=retry RPC@366951813 returned Deferred@299936742(state=PENDING, result=null, callback=(continuation of Deferred@1424111550 after retry RPC@366951813), errback=(continuation of Deferred@1424111550 after retry RPC@366951813)), so the following Deferred is getting paused: Deferred@1424111550(state=PAUSED, result=Deferred@299936742, callback=org.apache.kudu.client.AsyncKuduClient$6@92341c4 -> wakeup thread pool-3-thread-2, errback=passthrough -> wakeup thread pool-3-thread-2) 01:26:50.569 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.573 [DEBUG - New I/O worker #1147] (Negotiator.java:764) Authenticated connection [id: 0x80f99b95, /127.1.193.124:46378 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.573 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.575 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.575 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.576 [DEBUG - New I/O worker #993] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.578 [DEBUG - New I/O worker #1164] (Negotiator.java:764) Authenticated connection [id: 0x24d0cf4c, /127.1.193.126:51192 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.577 [DEBUG - New I/O worker #979] (Negotiator.java:764) Authenticated connection [id: 0x5e9616f2, /127.1.193.125:50453 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.579 [DEBUG - New I/O worker #993] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.582 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.583 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=332), Traces: [0ms] refreshing cache from master, [311ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [312ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [312ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [330ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [332ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [332ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.583 [DEBUG - New I/O worker #980] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.583 [DEBUG - New I/O worker #983] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.584 [DEBUG - New I/O worker #1144] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.584 [DEBUG - New I/O worker #1169] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.583 [DEBUG - New I/O worker #979] (Deferred.java:1330) callback=retry RPC@1841272883 returned Deferred@1769996636(state=PENDING, result=null, callback=(continuation of Deferred@746754568 after retry RPC@1841272883), errback=(continuation of Deferred@746754568 after retry RPC@1841272883)), so the following Deferred is getting paused: Deferred@746754568(state=PAUSED, result=Deferred@1769996636, callback=org.apache.kudu.client.AsyncKuduClient$6@c0ba4be -> wakeup thread pool-3-thread-40, errback=passthrough -> wakeup thread pool-3-thread-40) 01:26:50.584 [DEBUG - pool-3-thread-2] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.585 [DEBUG - New I/O worker #978] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.587 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.591 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.591 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.591 [DEBUG - New I/O worker #980] (Negotiator.java:764) Authenticated connection [id: 0xd9d3d2e0, /127.1.193.124:46381 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.592 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.592 [DEBUG - New I/O worker #979] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.593 [DEBUG - New I/O worker #979] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.593 [DEBUG - New I/O worker #993] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.594 [DEBUG - pool-3-thread-40] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.594 [DEBUG - New I/O worker #1164] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.594 [DEBUG - New I/O worker #979] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.594 [DEBUG - New I/O worker #1147] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.596 [DEBUG - New I/O worker #983] (Negotiator.java:764) Authenticated connection [id: 0xea19b81a, /127.1.193.126:51195 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.597 [DEBUG - New I/O worker #1143] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.597 [DEBUG - New I/O worker #1298] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.597 [DEBUG - New I/O worker #987] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.855 [DEBUG - New I/O worker #978] (Negotiator.java:764) Authenticated connection [id: 0x8e045dac, /127.1.193.125:50456 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.861 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.861 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=617), Traces: [0ms] refreshing cache from master, [336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [357ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [617ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.862 [DEBUG - New I/O worker #978] (Deferred.java:1330) callback=retry RPC@1651083013 returned Deferred@1716920208(state=PENDING, result=null, callback=(continuation of Deferred@592743742 after retry RPC@1651083013), errback=(continuation of Deferred@592743742 after retry RPC@1651083013)), so the following Deferred is getting paused: Deferred@592743742(state=PAUSED, result=Deferred@1716920208, callback=org.apache.kudu.client.AsyncKuduClient$6@3c42e4c0 -> wakeup thread pool-3-thread-70, errback=passthrough -> wakeup thread pool-3-thread-70) 01:26:50.863 [DEBUG - New I/O worker #1143] (Negotiator.java:764) Authenticated connection [id: 0x0059b84b, /127.1.193.124:46384 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.866 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.867 [DEBUG - New I/O worker #987] (Negotiator.java:764) Authenticated connection [id: 0x7e8290c9, /127.1.193.125:50459 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.867 [DEBUG - New I/O worker #1298] (Negotiator.java:764) Authenticated connection [id: 0x5d40fece, /127.1.193.126:51198 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.870 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.871 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=644), Traces: [0ms] refreshing cache from master, [368ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [368ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [369ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.871 [DEBUG - New I/O worker #987] (Deferred.java:1330) callback=retry RPC@499218887 returned Deferred@436137639(state=PENDING, result=null, callback=(continuation of Deferred@1560048636 after retry RPC@499218887), errback=(continuation of Deferred@1560048636 after retry RPC@499218887)), so the following Deferred is getting paused: Deferred@1560048636(state=PAUSED, result=Deferred@436137639, callback=org.apache.kudu.client.AsyncKuduClient$6@7bf37d89 -> wakeup thread pool-3-thread-49, errback=passthrough -> wakeup thread pool-3-thread-49) 01:26:50.872 [DEBUG - New I/O worker #1132] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.874 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.876 [DEBUG - New I/O worker #1002] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.877 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.877 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.878 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.878 [DEBUG - New I/O worker #978] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.878 [DEBUG - New I/O worker #978] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.879 [DEBUG - pool-3-thread-70] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.880 [DEBUG - New I/O worker #983] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.880 [DEBUG - New I/O worker #1132] (Negotiator.java:764) Authenticated connection [id: 0x54ff58a0, /127.1.193.124:46390 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.880522 1948 connection.cc:647] server connection from 127.1.193.124:46381 recv error: Network error: recv error from 0.0.0.0:0: Transport endpoint is not connected (error 107) 01:26:50.880 [DEBUG - New I/O worker #978] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.882 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.882 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.883 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.883 [DEBUG - New I/O worker #987] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.884 [DEBUG - New I/O worker #987] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.885 [DEBUG - pool-3-thread-49] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.885 [DEBUG - New I/O worker #1298] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:50.886 [DEBUG - New I/O worker #1143] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.885 [DEBUG - pool-3-thread-43] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.886 [DEBUG - New I/O worker #1002] (Negotiator.java:764) Authenticated connection [id: 0x018e3e3e, /127.1.193.125:50465 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.886 [DEBUG - New I/O worker #987] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.888 [DEBUG - pool-3-thread-10] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.889 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.889 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=345), Traces: [0ms] refreshing cache from master, [326ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [326ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [340ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [345ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.890 [DEBUG - New I/O worker #1002] (Deferred.java:1330) callback=retry RPC@1747205634 returned Deferred@1813755870(state=PENDING, result=null, callback=(continuation of Deferred@689517625 after retry RPC@1747205634), errback=(continuation of Deferred@689517625 after retry RPC@1747205634)), so the following Deferred is getting paused: Deferred@689517625(state=PAUSED, result=Deferred@1813755870, callback=org.apache.kudu.client.AsyncKuduClient$6@51e79b18 -> wakeup thread pool-3-thread-80, errback=passthrough -> wakeup thread pool-3-thread-80) 01:26:50.890 [DEBUG - pool-3-thread-5] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.892 [DEBUG - New I/O worker #1026] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.895 [DEBUG - New I/O worker #1059] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.900 [DEBUG - New I/O worker #1026] (Negotiator.java:764) Authenticated connection [id: 0x5bf5dee7, /127.1.193.126:51207 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.907 [DEBUG - New I/O worker #1059] (Negotiator.java:764) Authenticated connection [id: 0xc63a33ca, /127.1.193.125:50471 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.910 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.911 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=430), Traces: [0ms] refreshing cache from master, [411ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [411ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [411ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [430ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.911 [DEBUG - New I/O worker #1059] (Deferred.java:1330) callback=retry RPC@2106524152 returned Deferred@2041612945(state=PENDING, result=null, callback=(continuation of Deferred@1018301379 after retry RPC@2106524152), errback=(continuation of Deferred@1018301379 after retry RPC@2106524152)), so the following Deferred is getting paused: Deferred@1018301379(state=PAUSED, result=Deferred@2041612945, callback=org.apache.kudu.client.AsyncKuduClient$6@841a25d -> wakeup thread pool-3-thread-65, errback=passthrough -> wakeup thread pool-3-thread-65) 01:26:50.912 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.915 [DEBUG - New I/O worker #1028] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.916 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.918 [DEBUG - New I/O worker #1013] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:50.919 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.919 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.919 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.920 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.921 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.921 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.922 [DEBUG - New I/O worker #1002] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.922 [DEBUG - New I/O worker #1002] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.923 [DEBUG - New I/O worker #1002] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.923 [DEBUG - New I/O worker #1132] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:50.923 [DEBUG - pool-3-thread-80] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.924 [DEBUG - New I/O worker #1059] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.924 [DEBUG - New I/O worker #1059] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.924 [DEBUG - New I/O worker #1028] (Negotiator.java:764) Authenticated connection [id: 0xb7ada23f, /127.1.193.125:50474 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.925 [DEBUG - New I/O worker #1013] (Negotiator.java:764) Authenticated connection [id: 0x7131e7ae, /127.1.193.125:50477 => /127.1.193.125:37435] using SASL/PLAIN 01:26:50.930 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.931 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=458), Traces: [0ms] refreshing cache from master, [442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [458ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.935 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:50.935 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=462), Traces: [0ms] refreshing cache from master, [419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [461ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.935212 4231 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.870715 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51204 01:26:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.871274 (+ 559us) server_negotiation.cc:184] Beginning negotiation 01:26:50.936 [DEBUG - New I/O worker #1028] (Deferred.java:1330) callback=retry RPC@720627459 returned Deferred@785151621(state=PENDING, result=null, callback=(continuation of Deferred@1808723256 after retry RPC@720627459), errback=(continuation of Deferred@1808723256 after retry RPC@720627459)), so the following Deferred is getting paused: Deferred@1808723256(state=PAUSED, result=Deferred@785151621, callback=org.apache.kudu.client.AsyncKuduClient$6@6f919e01 -> wakeup thread pool-3-thread-16, errback=passthrough -> wakeup thread pool-3-thread-16) 01:26:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.871302 (+ 28us) server_negotiation.cc:373] Waiting for connection header 01:26:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.923813 (+ 52511us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51204: BlockingRecv error: recv got EOF from 127.1.193.126:51204 (error 108) 01:26:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":286} 01:26:50.938 [DEBUG - New I/O worker #1115] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.938 [DEBUG - New I/O worker #1013] (Deferred.java:1330) callback=retry RPC@1927944068 returned Deferred@1993710849(state=PENDING, result=null, callback=(continuation of Deferred@869735871 after retry RPC@1927944068), errback=(continuation of Deferred@869735871 after retry RPC@1927944068)), so the following Deferred is getting paused: Deferred@869735871(state=PAUSED, result=Deferred@1993710849, callback=org.apache.kudu.client.AsyncKuduClient$6@20dafbca -> wakeup thread pool-3-thread-62, errback=passthrough -> wakeup thread pool-3-thread-62) 01:26:50.941 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.942 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:50.946 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.946 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.946 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.946 [DEBUG - New I/O worker #1013] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.947 [DEBUG - New I/O worker #1013] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.947 [DEBUG - pool-3-thread-62] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.948 [DEBUG - New I/O worker #1013] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.950856 4561 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.935859 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51216 01:26:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.937523 (+ 1664us) server_negotiation.cc:184] Beginning negotiation 01:26:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.937550 (+ 27us) server_negotiation.cc:373] Waiting for connection header 01:26:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.950057 (+ 12507us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51216: BlockingRecv error: recv got EOF from 127.1.193.126:51216 (error 108) 01:26:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":1429} 01:26:50.951 [DEBUG - New I/O worker #1045] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.953050 4567 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.916213 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46402 01:26:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.921040 (+ 4827us) server_negotiation.cc:184] Beginning negotiation 01:26:50.953 [DEBUG - New I/O worker #1112] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.921066 (+ 26us) server_negotiation.cc:373] Waiting for connection header 01:26:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.950848 (+ 29782us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46402: BlockingRecv error: recv got EOF from 127.1.193.124:46402 (error 108) 01:26:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":4598,"thread_start_us":4975,"threads_started":1} 01:26:50.954 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:50.954 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:50.954 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:50.954 [DEBUG - New I/O worker #1028] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:50.955 [DEBUG - New I/O worker #1028] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:50.955 [DEBUG - pool-3-thread-16] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.956 [DEBUG - New I/O worker #1115] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: connection disconnected 01:26:50.957 [DEBUG - New I/O worker #1059] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.957 [DEBUG - pool-3-thread-65] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.956812 4560 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.893513 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51213 01:26:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.935523 (+ 42010us) server_negotiation.cc:184] Beginning negotiation 01:26:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.935557 (+ 34us) server_negotiation.cc:373] Waiting for connection header 01:26:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.956030 (+ 20473us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51213: BlockingRecv error: recv got EOF from 127.1.193.126:51213 (error 108) 01:26:50.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":41660,"thread_start_us":40866,"threads_started":1} 01:26:50.958 [DEBUG - New I/O worker #1028] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:50.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.958168 3901 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.894712 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46396 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.895127 (+ 415us) server_negotiation.cc:184] Beginning negotiation 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.895154 (+ 27us) server_negotiation.cc:373] Waiting for connection header 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.957478 (+ 62324us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46396: BlockingRecv error: recv got EOF from 127.1.193.124:46396 (error 108) 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":159} 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.959522 4458 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.892870 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51210 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.902015 (+ 9145us) server_negotiation.cc:184] Beginning negotiation 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.902040 (+ 25us) server_negotiation.cc:373] Waiting for connection header 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.958789 (+ 56749us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51210: BlockingRecv error: recv got EOF from 127.1.193.126:51210 (error 108) 01:26:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":8931,"thread_start_us":42611,"threads_started":1} 01:26:50.965 [DEBUG - New I/O worker #1046] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:50.965643 4562 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.895399 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46399 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.914560 (+ 19161us) server_negotiation.cc:184] Beginning negotiation 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.914586 (+ 26us) server_negotiation.cc:373] Waiting for connection header 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.914628 (+ 42us) server_negotiation.cc:381] Connection header received 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.914742 (+ 114us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.914765 (+ 23us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.914940 (+ 175us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.940070 (+ 25130us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.940379 (+ 309us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.940954 (+ 575us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.943609 (+ 2655us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.955267 (+ 11658us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:50.962265 (+ 6998us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46399: BlockingWrite error: write error: Connection reset by peer (error 104) 01:26:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":18910,"thread_start_us":35238,"threads_started":1} 01:26:50.966 [DEBUG - pool-3-thread-14] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.968 [DEBUG - New I/O worker #1045] (Negotiator.java:764) Authenticated connection [id: 0xe5501f46, /127.1.193.124:46405 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.972 [DEBUG - New I/O worker #1112] (Negotiator.java:764) Authenticated connection [id: 0x363eb955, /127.1.193.126:51225 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.972 [DEBUG - New I/O worker #1046] (Negotiator.java:764) Authenticated connection [id: 0x4a9090b8, /127.1.193.124:46411 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.975 [DEBUG - pool-3-thread-57] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.978 [DEBUG - New I/O worker #1041] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:50.978 [DEBUG - New I/O worker #1198] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:50.985 [DEBUG - New I/O worker #1041] (Negotiator.java:764) Authenticated connection [id: 0x8888ef13, /127.1.193.124:46417 => /127.1.193.124:58247] using SASL/PLAIN 01:26:50.986 [DEBUG - New I/O worker #1198] (Negotiator.java:764) Authenticated connection [id: 0x5f39b6fd, /127.1.193.126:51231 => /127.1.193.126:40739] using SASL/PLAIN 01:26:50.988 [DEBUG - pool-3-thread-24] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:50.956 [INFO - New I/O worker #1115] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:51.023 [DEBUG - New I/O worker #1116] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.024 [DEBUG - New I/O worker #1037] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.025 [DEBUG - New I/O worker #1197] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.025 [DEBUG - New I/O worker #1190] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.029 [DEBUG - New I/O worker #1033] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.032 [DEBUG - New I/O worker #1116] (Negotiator.java:764) Authenticated connection [id: 0x4ae75b6b, /127.1.193.124:46421 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.035 [DEBUG - New I/O worker #1197] (Negotiator.java:764) Authenticated connection [id: 0x2741b1d9, /127.1.193.124:46422 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.033 [DEBUG - New I/O worker #1190] (Negotiator.java:764) Authenticated connection [id: 0xacc8c389, /127.1.193.126:51236 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.038 [DEBUG - New I/O worker #1033] (Negotiator.java:764) Authenticated connection [id: 0xaa77fe42, /127.1.193.125:50496 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.042 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.043 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=155), Traces: [0ms] refreshing cache from master, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [150ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [154ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.044 [DEBUG - New I/O worker #1033] (Deferred.java:1330) callback=retry RPC@1790108944 returned Deferred@1854762142(state=PENDING, result=null, callback=(continuation of Deferred@730853163 after retry RPC@1790108944), errback=(continuation of Deferred@730853163 after retry RPC@1790108944)), so the following Deferred is getting paused: Deferred@730853163(state=PAUSED, result=Deferred@1854762142, callback=org.apache.kudu.client.AsyncKuduClient$6@6789d7c3 -> wakeup thread pool-3-thread-10, errback=passthrough -> wakeup thread pool-3-thread-10) 01:26:51.047 [DEBUG - pool-3-thread-78] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.048 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.052 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.052 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.052 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.053 [DEBUG - New I/O worker #1033] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.053 [DEBUG - New I/O worker #1033] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.054 [DEBUG - pool-3-thread-10] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.055 [DEBUG - New I/O worker #1033] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.055 [DEBUG - New I/O worker #1197] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.055636 4458 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.022035 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51237 01:26:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.022794 (+ 759us) server_negotiation.cc:184] Beginning negotiation 01:26:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.022819 (+ 25us) server_negotiation.cc:373] Waiting for connection header 01:26:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.055040 (+ 32221us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51237: BlockingRecv error: recv got EOF from 127.1.193.126:51237 (error 108) 01:26:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":525} 01:26:51.059 [DEBUG - New I/O worker #996] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.062 [DEBUG - New I/O worker #991] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.063 [DEBUG - New I/O worker #1037] (Negotiator.java:764) Authenticated connection [id: 0xc0b7ff30, /127.1.193.125:50495 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.064 [DEBUG - pool-3-thread-56] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.066 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.067 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=175), Traces: [0ms] refreshing cache from master, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [145ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [175ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.067 [DEBUG - pool-3-thread-29] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.067 [DEBUG - New I/O worker #1037] (Deferred.java:1330) callback=retry RPC@2022243674 returned Deferred@2092394131(state=PENDING, result=null, callback=(continuation of Deferred@968226657 after retry RPC@2022243674), errback=(continuation of Deferred@968226657 after retry RPC@2022243674)), so the following Deferred is getting paused: Deferred@968226657(state=PAUSED, result=Deferred@2092394131, callback=org.apache.kudu.client.AsyncKuduClient$6@379bd5a1 -> wakeup thread pool-3-thread-5, errback=passthrough -> wakeup thread pool-3-thread-5) 01:26:51.070 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.070 [DEBUG - New I/O worker #996] (Negotiator.java:764) Authenticated connection [id: 0x936501cf, /127.1.193.125:50501 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.073 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.073 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=187), Traces: [0ms] refreshing cache from master, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.074 [DEBUG - New I/O worker #996] (Deferred.java:1330) callback=retry RPC@852657759 returned Deferred@921507037(state=PENDING, result=null, callback=(continuation of Deferred@1945078884 after retry RPC@852657759), errback=(continuation of Deferred@1945078884 after retry RPC@852657759)), so the following Deferred is getting paused: Deferred@1945078884(state=PAUSED, result=Deferred@921507037, callback=org.apache.kudu.client.AsyncKuduClient$6@64906da3 -> wakeup thread pool-3-thread-43, errback=passthrough -> wakeup thread pool-3-thread-43) 01:26:51.074 [DEBUG - New I/O worker #991] (Negotiator.java:764) Authenticated connection [id: 0xb64bd324, /127.1.193.125:50504 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.074 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.076 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.077 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.077 [DEBUG - New I/O worker #1037] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.077 [DEBUG - New I/O worker #1037] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.078 [DEBUG - pool-3-thread-5] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.079 [DEBUG - New I/O worker #1116] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.079 [DEBUG - New I/O worker #1190] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.081 [DEBUG - New I/O worker #1296] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.081 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.081 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=34), Traces: [0ms] refreshing cache from master, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [34ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.076 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.109 [DEBUG - pool-3-thread-64] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.116 [DEBUG - New I/O worker #1024] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.108 [DEBUG - New I/O worker #980] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.117 [INFO - New I/O worker #980] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:51.108 [DEBUG - New I/O worker #1020] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.107 [DEBUG - New I/O worker #1091] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.107 [DEBUG - New I/O worker #966] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.106 [DEBUG - New I/O worker #984] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.104 [DEBUG - New I/O worker #992] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.104 [DEBUG - New I/O worker #1230] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.103 [DEBUG - New I/O worker #1255] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.127 [INFO - New I/O worker #1255] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: java.nio.channels.ClosedChannelException 01:26:51.099 [DEBUG - New I/O worker #1378] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.128 [INFO - New I/O worker #1378] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: java.nio.channels.ClosedChannelException 01:26:51.095 [DEBUG - New I/O worker #1106] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.129 [INFO - New I/O worker #1106] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: java.nio.channels.ClosedChannelException 01:26:51.129 [DEBUG - New I/O worker #1025] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.130 [DEBUG - New I/O worker #1148] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.095 [DEBUG - New I/O worker #1191] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.095 [DEBUG - New I/O worker #1276] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.134 [DEBUG - New I/O worker #968] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.134 [DEBUG - New I/O worker #1156] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.135 [DEBUG - New I/O worker #1149] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.095 [DEBUG - New I/O worker #1179] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.136 [INFO - New I/O worker #1179] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: java.nio.channels.ClosedChannelException 01:26:51.092 [DEBUG - New I/O worker #1094] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.091 [DEBUG - New I/O worker #1093] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.091 [DEBUG - New I/O worker #1079] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.090 [DEBUG - New I/O worker #1076] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.088 [DEBUG - New I/O worker #1173] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state CONNECTING due to: java.nio.channels.ClosedChannelException 01:26:51.141 [INFO - New I/O worker #1173] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: java.nio.channels.ClosedChannelException 01:26:51.082 [DEBUG - New I/O worker #1170] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.082 [DEBUG - New I/O worker #991] (Deferred.java:1330) callback=retry RPC@296746582 returned Deferred@361785531(state=PENDING, result=null, callback=(continuation of Deferred@1351808109 after retry RPC@296746582), errback=(continuation of Deferred@1351808109 after retry RPC@296746582)), so the following Deferred is getting paused: Deferred@1351808109(state=PAUSED, result=Deferred@361785531, callback=org.apache.kudu.client.AsyncKuduClient$6@fbbc3ef -> wakeup thread pool-3-thread-78, errback=passthrough -> wakeup thread pool-3-thread-78) 01:26:51.078 [DEBUG - New I/O worker #1037] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.142 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.142 [DEBUG - New I/O worker #992] (Negotiator.java:764) Authenticated connection [id: 0xfa779f3c, /127.1.193.125:50486 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.140 [DEBUG - New I/O worker #1201] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.134 [INFO - New I/O worker #1276] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: java.nio.channels.ClosedChannelException 01:26:51.131 [INFO - New I/O worker #1191] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: java.nio.channels.ClosedChannelException 01:26:51.146 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.146 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=674), Traces: [0ms] refreshing cache from master, [467ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [467ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [467ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [504ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [504ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [674ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.147 [DEBUG - New I/O worker #984] (Negotiator.java:764) Authenticated connection [id: 0x34bfde45, /127.1.193.125:50480 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.147 [DEBUG - New I/O worker #1091] (Negotiator.java:764) Authenticated connection [id: 0x543ae3ca, /127.1.193.124:46414 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.126 [DEBUG - New I/O worker #1296] (Negotiator.java:764) Authenticated connection [id: 0x124e2459, /127.1.193.126:51243 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.123 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.148 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.148 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.148 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.149 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.149 [DEBUG - New I/O worker #991] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.149 [DEBUG - New I/O worker #991] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.147 [DEBUG - New I/O worker #1020] (Negotiator.java:764) Authenticated connection [id: 0xf405caca, /127.1.193.125:50468 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.148 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.150 [DEBUG - New I/O worker #974] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.150 [DEBUG - New I/O worker #996] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.151 [DEBUG - New I/O worker #996] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.151 [DEBUG - pool-3-thread-78] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.152 [DEBUG - pool-3-thread-43] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.153254 3901 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.052047 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46429 01:26:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.052547 (+ 500us) server_negotiation.cc:184] Beginning negotiation 01:26:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.052577 (+ 30us) server_negotiation.cc:373] Waiting for connection header 01:26:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080081 (+ 27504us) server_negotiation.cc:381] Connection header received 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080215 (+ 134us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080243 (+ 28us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080381 (+ 138us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080577 (+ 196us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.152316 (+ 71739us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46429: BlockingRecv error: recv got EOF from 127.1.193.124:46429 (error 108) 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":271} 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.153586 4561 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.040155 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51240 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.040516 (+ 361us) server_negotiation.cc:184] Beginning negotiation 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.040543 (+ 27us) server_negotiation.cc:373] Waiting for connection header 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.079802 (+ 39259us) server_negotiation.cc:381] Connection header received 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080008 (+ 206us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080041 (+ 33us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080236 (+ 195us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080404 (+ 168us) server_negotiation.cc:205] Negotiated authn=TOKEN 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.084252 (+ 3848us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.084858 (+ 606us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.152587 (+ 67729us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51240: BlockingRecv error: recv got EOF from 127.1.193.126:51240 (error 108) 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":134} 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.154963 4568 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.040380 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46426 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.041388 (+ 1008us) server_negotiation.cc:184] Beginning negotiation 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.041409 (+ 21us) server_negotiation.cc:373] Waiting for connection header 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.079632 (+ 38223us) server_negotiation.cc:381] Connection header received 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.079880 (+ 248us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.079912 (+ 32us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080118 (+ 206us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.080302 (+ 184us) server_negotiation.cc:205] Negotiated authn=TOKEN 01:26:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.083096 (+ 2794us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.083719 (+ 623us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.153881 (+ 70162us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46426: BlockingRecv error: recv got EOF from 127.1.193.124:46426 (error 108) 01:26:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":796} 01:26:51.155 [INFO - New I/O worker #1170] (Connection.java:435) [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:51.159 [DEBUG - New I/O worker #1170] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:51.159 [INFO - New I/O worker #1170] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:51.160 [DEBUG - New I/O worker #1230] (Negotiator.java:764) Authenticated connection [id: 0xc765f579, /127.1.193.126:51228 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.156184 1815 connection.cc:554] Connection torn down before Call kudu.master.MasterService.ConnectToMaster from 127.1.193.126:51243 (request call id 0) could send its response 01:26:51.157 [DEBUG - New I/O worker #992] (Deferred.java:1330) callback=retry RPC@818436144 returned Deferred@888654577(state=PENDING, result=null, callback=(continuation of Deferred@1911902731 after retry RPC@818436144), errback=(continuation of Deferred@1911902731 after retry RPC@818436144)), so the following Deferred is getting paused: Deferred@1911902731(state=PAUSED, result=Deferred@888654577, callback=org.apache.kudu.client.AsyncKuduClient$6@32b5e61d -> wakeup thread pool-3-thread-75, errback=passthrough -> wakeup thread pool-3-thread-75) 01:26:51.157 [DEBUG - New I/O worker #991] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.157 [DEBUG - New I/O worker #1296] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.161 [INFO - New I/O worker #1296] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: connection disconnected 01:26:51.156 [DEBUG - New I/O worker #966] (Negotiator.java:764) Authenticated connection [id: 0x8605d258, /127.1.193.125:50507 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.155 [DEBUG - New I/O worker #996] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.165 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.165 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=674), Traces: [0ms] refreshing cache from master, [399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [400ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [400ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [413ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [674ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.165 [DEBUG - New I/O worker #1024] (Negotiator.java:764) Authenticated connection [id: 0x3b3f26b3, /127.1.193.124:46393 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.165 [DEBUG - New I/O worker #1020] (Deferred.java:1330) callback=retry RPC@1364453476 returned Deferred@1433171606(state=PENDING, result=null, callback=(continuation of Deferred@275702367 after retry RPC@1364453476), errback=(continuation of Deferred@275702367 after retry RPC@1364453476)), so the following Deferred is getting paused: Deferred@275702367(state=PAUSED, result=Deferred@1433171606, callback=org.apache.kudu.client.AsyncKuduClient$6@6feaa699 -> wakeup thread pool-3-thread-47, errback=passthrough -> wakeup thread pool-3-thread-47) 01:26:51.167 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.167 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=99), Traces: [0ms] refreshing cache from master, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [99ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.167 [DEBUG - New I/O worker #966] (Deferred.java:1330) callback=retry RPC@1288281727 returned Deferred@1224086334(state=PENDING, result=null, callback=(continuation of Deferred@234132548 after retry RPC@1288281727), errback=(continuation of Deferred@234132548 after retry RPC@1288281727)), so the following Deferred is getting paused: Deferred@234132548(state=PAUSED, result=Deferred@1224086334, callback=org.apache.kudu.client.AsyncKuduClient$6@3c8951bc -> wakeup thread pool-3-thread-29, errback=passthrough -> wakeup thread pool-3-thread-29) 01:26:51.175 [DEBUG - New I/O worker #1148] (Negotiator.java:764) Authenticated connection [id: 0xe3d477da, /127.1.193.124:46387 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.176 [DEBUG - New I/O worker #1149] (Negotiator.java:764) Authenticated connection [id: 0xedc18aaf, /127.1.193.126:51201 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.177 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.177 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=705), Traces: [0ms] refreshing cache from master, [453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [499ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.178 [DEBUG - New I/O worker #968] (Negotiator.java:764) Authenticated connection [id: 0x9a8d2669, /127.1.193.124:46432 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.178 [DEBUG - New I/O worker #984] (Deferred.java:1330) callback=retry RPC@134454854 returned Deferred@205261726(state=PENDING, result=null, callback=(continuation of Deferred@1228838013 after retry RPC@134454854), errback=(continuation of Deferred@1228838013 after retry RPC@134454854)), so the following Deferred is getting paused: Deferred@1228838013(state=PAUSED, result=Deferred@205261726, callback=org.apache.kudu.client.AsyncKuduClient$6@6cd85ea4 -> wakeup thread pool-3-thread-60, errback=passthrough -> wakeup thread pool-3-thread-60) 01:26:51.179 [DEBUG - New I/O worker #1079] (Negotiator.java:764) Authenticated connection [id: 0xd403f8ad, /127.1.193.125:50489 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.180 [DEBUG - New I/O worker #1093] (Negotiator.java:764) Authenticated connection [id: 0x271df4f5, /127.1.193.124:46408 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.181 [DEBUG - New I/O worker #1076] (Negotiator.java:764) Authenticated connection [id: 0x80c5467e, /127.1.193.125:50483 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.183 [DEBUG - New I/O worker #1094] (Negotiator.java:764) Authenticated connection [id: 0x022b25c5, /127.1.193.126:51222 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.184 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.184 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=713), Traces: [0ms] refreshing cache from master, [493ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [493ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [493ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.185 [DEBUG - New I/O worker #1079] (Deferred.java:1330) callback=retry RPC@1439397030 returned Deferred@1375009479(state=PENDING, result=null, callback=(continuation of Deferred@351698589 after retry RPC@1439397030), errback=(continuation of Deferred@351698589 after retry RPC@1439397030)), so the following Deferred is getting paused: Deferred@351698589(state=PAUSED, result=Deferred@1375009479, callback=org.apache.kudu.client.AsyncKuduClient$6@2bc8d49d -> wakeup thread pool-3-thread-18, errback=passthrough -> wakeup thread pool-3-thread-18) 01:26:51.185 [DEBUG - New I/O worker #1201] (Negotiator.java:764) Authenticated connection [id: 0x20be7142, /127.1.193.126:51219 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.187 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.188 [DEBUG - New I/O worker #1025] (Negotiator.java:764) Authenticated connection [id: 0xf8a989ff, /127.1.193.125:50492 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.190 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.191 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.192 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.193 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=218), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [13ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [14ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [217ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.194 [DEBUG - New I/O worker #1025] (Deferred.java:1330) callback=retry RPC@318173047 returned Deferred@382302705(state=PENDING, result=null, callback=(continuation of Deferred@1405871436 after retry RPC@318173047), errback=(continuation of Deferred@1405871436 after retry RPC@318173047)), so the following Deferred is getting paused: Deferred@1405871436(state=PAUSED, result=Deferred@382302705, callback=org.apache.kudu.client.AsyncKuduClient$6@47fa40ae -> wakeup thread pool-3-thread-57, errback=passthrough -> wakeup thread pool-3-thread-57) 01:26:51.195 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.196 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=723), Traces: [0ms] refreshing cache from master, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [713ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [715ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.196 [DEBUG - New I/O worker #1076] (Deferred.java:1330) callback=retry RPC@900407169 returned Deferred@831819244(state=PENDING, result=null, callback=(continuation of Deferred@1955993018 after retry RPC@900407169), errback=(continuation of Deferred@1955993018 after retry RPC@900407169)), so the following Deferred is getting paused: Deferred@1955993018(state=PAUSED, result=Deferred@831819244, callback=org.apache.kudu.client.AsyncKuduClient$6@289255b2 -> wakeup thread pool-3-thread-69, errback=passthrough -> wakeup thread pool-3-thread-69) 01:26:51.197 [DEBUG - New I/O worker #1155] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.198 [DEBUG - New I/O worker #967] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.192 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.200 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.200 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.200 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.201 [DEBUG - New I/O worker #1020] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.201 [DEBUG - New I/O worker #1020] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.202 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.203 [DEBUG - New I/O worker #1020] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.205 [DEBUG - New I/O worker #1024] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.196 [DEBUG - New I/O worker #1156] (Negotiator.java:764) Authenticated connection [id: 0x998a515e, /127.1.193.126:51246 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.194 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.207 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.208 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.208 [DEBUG - New I/O worker #966] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.208 [DEBUG - New I/O worker #966] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.192 [DEBUG - New I/O worker #974] (Negotiator.java:764) Authenticated connection [id: 0xc6f4fe74, /127.1.193.125:50462 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.202 [DEBUG - New I/O worker #1026] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.209 [DEBUG - pool-3-thread-29] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.202 [DEBUG - pool-3-thread-47] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.201 [DEBUG - New I/O worker #969] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.210 [DEBUG - New I/O worker #968] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.200 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.210 [DEBUG - New I/O worker #1156] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.211 [DEBUG - New I/O worker #966] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.213 [DEBUG - New I/O worker #1155] (Negotiator.java:764) Authenticated connection [id: 0xd90bbbc7, /127.1.193.126:51249 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.218 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.218 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.218 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.218 [DEBUG - New I/O worker #967] (Negotiator.java:764) Authenticated connection [id: 0x2586a880, /127.1.193.125:50510 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.219 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.219 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.218 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.220 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.221 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.221 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.221 [DEBUG - New I/O worker #984] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.222 [DEBUG - New I/O worker #984] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.223 [DEBUG - pool-3-thread-60] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.220 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.223 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.223 [DEBUG - New I/O worker #1025] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.224 [DEBUG - New I/O worker #1025] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.225 [DEBUG - New I/O worker #1201] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.225 [DEBUG - pool-3-thread-57] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.226 [DEBUG - New I/O worker #1041] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.226 [DEBUG - New I/O worker #1198] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.219 [DEBUG - New I/O worker #1079] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.223 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.223 [DEBUG - New I/O worker #984] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.223 [DEBUG - New I/O worker #1045] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.220 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=992), Traces: [0ms] refreshing cache from master, [627ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [954ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [960ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [989ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.228 [DEBUG - New I/O worker #1317] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.227 [DEBUG - New I/O worker #1025] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.227 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.227 [DEBUG - New I/O worker #1079] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.230 [DEBUG - New I/O worker #1159] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.229 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.230 [DEBUG - New I/O worker #1076] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.231 [DEBUG - New I/O worker #1076] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.231 [DEBUG - New I/O worker #964] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.232 [DEBUG - New I/O worker #1230] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.232 [DEBUG - New I/O worker #1076] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.228 [DEBUG - New I/O worker #974] (Deferred.java:1330) callback=retry RPC@482461801 returned Deferred@419323689(state=PENDING, result=null, callback=(continuation of Deferred@1576845906 after retry RPC@482461801), errback=(continuation of Deferred@1576845906 after retry RPC@482461801)), so the following Deferred is getting paused: Deferred@1576845906(state=PAUSED, result=Deferred@419323689, callback=org.apache.kudu.client.AsyncKuduClient$6@7e7b8148 -> wakeup thread pool-3-thread-45, errback=passthrough -> wakeup thread pool-3-thread-45) 01:26:51.232 [DEBUG - pool-3-thread-69] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.233 [DEBUG - New I/O worker #1094] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.234 [DEBUG - New I/O worker #969] (Negotiator.java:764) Authenticated connection [id: 0xfdb2df28, /127.1.193.124:46435 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.232 [DEBUG - New I/O worker #1093] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.230 [DEBUG - New I/O worker #1079] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.230 [DEBUG - New I/O worker #1091] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.230 [DEBUG - pool-3-thread-18] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.237 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.237 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=120), Traces: [0ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [102ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [120ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:51.237 [DEBUG - New I/O worker #967] (Deferred.java:1330) callback=retry RPC@1227809479 returned Deferred@1292992314(state=PENDING, result=null, callback=(continuation of Deferred@135522556 after retry RPC@1227809479), errback=(continuation of Deferred@135522556 after retry RPC@1227809479)), so the following Deferred is getting paused: Deferred@135522556(state=PAUSED, result=Deferred@1292992314, callback=org.apache.kudu.client.AsyncKuduClient$6@757f6f0 -> wakeup thread pool-3-thread-64, errback=passthrough -> wakeup thread pool-3-thread-64) 01:26:51.238 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.239 [DEBUG - pool-3-thread-4] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.241 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.241 [DEBUG - New I/O worker #1159] (Negotiator.java:764) Authenticated connection [id: 0xcc82e111, /127.1.193.124:46438 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.244 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.245 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.245 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.245 [DEBUG - New I/O worker #974] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.245 [DEBUG - New I/O worker #974] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.246 [DEBUG - pool-3-thread-45] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.246 [DEBUG - New I/O worker #1148] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.247 [DEBUG - New I/O worker #974] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.246 [DEBUG - New I/O worker #1149] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.248 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.248 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.248 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.248 [DEBUG - pool-3-thread-67] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.249 [DEBUG - New I/O worker #1317] (Negotiator.java:764) Authenticated connection [id: 0xb615179d, /127.1.193.126:51252 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.248 [DEBUG - New I/O worker #992] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.249 [DEBUG - New I/O worker #992] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.250 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.250 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.250 [DEBUG - New I/O worker #1112] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.250 [DEBUG - pool-3-thread-75] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.250 [DEBUG - New I/O worker #1046] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.251 [DEBUG - New I/O worker #992] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.251 [DEBUG - New I/O worker #964] (Negotiator.java:764) Authenticated connection [id: 0x426c2594, /127.1.193.125:50513 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.251 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.251 [DEBUG - New I/O worker #967] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.252 [DEBUG - New I/O worker #967] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.254 [DEBUG - pool-3-thread-64] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.254 [DEBUG - New I/O worker #967] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.254 [DEBUG - New I/O worker #969] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.255 [DEBUG - New I/O worker #1155] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.257 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.257 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=192), Traces: [0ms] refreshing cache from master, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [179ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [190ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [192ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.258 [DEBUG - New I/O worker #964] (Deferred.java:1330) callback=retry RPC@778357281 returned Deferred@710630197(state=PENDING, result=null, callback=(continuation of Deferred@1868152858 after retry RPC@778357281), errback=(continuation of Deferred@1868152858 after retry RPC@778357281)), so the following Deferred is getting paused: Deferred@1868152858(state=PAUSED, result=Deferred@710630197, callback=org.apache.kudu.client.AsyncKuduClient$6@5641e4ac -> wakeup thread pool-3-thread-56, errback=passthrough -> wakeup thread pool-3-thread-56) 01:26:51.261 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.265 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.265 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.265 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.265 [DEBUG - New I/O worker #964] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.266 [DEBUG - New I/O worker #964] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.267 [DEBUG - pool-3-thread-56] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.267 [DEBUG - New I/O worker #1317] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.267 [DEBUG - New I/O worker #1159] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.268 [DEBUG - New I/O worker #964] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.272 [DEBUG - pool-3-thread-6] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.272 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.275 [DEBUG - pool-3-thread-28] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.276 [DEBUG - pool-3-thread-13] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.276 [DEBUG - New I/O worker #995] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.281 [DEBUG - pool-3-thread-59] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.282 [DEBUG - New I/O worker #1000] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.283 [DEBUG - New I/O worker #999] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.284 [DEBUG - New I/O worker #1075] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.278 [DEBUG - pool-3-thread-15] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.284 [DEBUG - New I/O worker #1072] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.285 [DEBUG - New I/O worker #1210] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.278 [DEBUG - New I/O worker #1069] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.277 [DEBUG - pool-3-thread-3] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.277 [DEBUG - pool-3-thread-1] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.286 [DEBUG - New I/O worker #1105] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.288 [DEBUG - New I/O worker #1061] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.297 [DEBUG - New I/O worker #1210] (Negotiator.java:764) Authenticated connection [id: 0x814e7b09, /127.1.193.126:51264 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.299 [DEBUG - New I/O worker #999] (Negotiator.java:764) Authenticated connection [id: 0x8dc57170, /127.1.193.124:46441 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.301 [DEBUG - New I/O worker #995] (Negotiator.java:764) Authenticated connection [id: 0x1f958e57, /127.1.193.125:50516 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.302 [DEBUG - pool-3-thread-63] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.303 [DEBUG - New I/O worker #1061] (Negotiator.java:764) Authenticated connection [id: 0x7b60a431, /127.1.193.125:50525 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.304 [DEBUG - New I/O worker #1105] (Negotiator.java:764) Authenticated connection [id: 0xf6eb5522, /127.1.193.124:46450 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.308 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.309 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=341), Traces: [0ms] refreshing cache from master, [306ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [306ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [307ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [337ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [341ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.309 [DEBUG - New I/O worker #1075] (Negotiator.java:764) Authenticated connection [id: 0x03c6a491, /127.1.193.126:51261 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.310 [DEBUG - New I/O worker #1072] (Negotiator.java:764) Authenticated connection [id: 0x2e31a85a, /127.1.193.124:46447 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.312 [DEBUG - New I/O worker #1043] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.312 [DEBUG - New I/O worker #1111] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.314 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.315 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=42), Traces: [0ms] refreshing cache from master, [9ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [27ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [36ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.315 [DEBUG - New I/O worker #1000] (Negotiator.java:764) Authenticated connection [id: 0xe248f832, /127.1.193.126:51255 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.316 [DEBUG - New I/O worker #1069] (Negotiator.java:764) Authenticated connection [id: 0x31e55570, /127.1.193.125:50519 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.317 [DEBUG - New I/O worker #1005] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.320 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.321 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=331), Traces: [0ms] refreshing cache from master, [285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [291ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [292ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [327ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [327ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [331ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.315 [DEBUG - New I/O worker #995] (Deferred.java:1330) callback=retry RPC@1920127791 returned Deferred@1984776633(state=PENDING, result=null, callback=(continuation of Deferred@860870932 after retry RPC@1920127791), errback=(continuation of Deferred@860870932 after retry RPC@1920127791)), so the following Deferred is getting paused: Deferred@860870932(state=PAUSED, result=Deferred@1984776633, callback=org.apache.kudu.client.AsyncKuduClient$6@663906de -> wakeup thread pool-3-thread-14, errback=passthrough -> wakeup thread pool-3-thread-14) 01:26:51.323 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.316 [DEBUG - New I/O worker #1061] (Deferred.java:1330) callback=retry RPC@628099638 returned Deferred@558864960(state=PENDING, result=null, callback=(continuation of Deferred@1682767885 after retry RPC@628099638), errback=(continuation of Deferred@1682767885 after retry RPC@628099638)), so the following Deferred is getting paused: Deferred@1682767885(state=PAUSED, result=Deferred@558864960, callback=org.apache.kudu.client.AsyncKuduClient$6@70e753c5 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:26:51.324 [DEBUG - New I/O worker #1069] (Deferred.java:1330) callback=retry RPC@890294189 returned Deferred@825187799(state=PENDING, result=null, callback=(continuation of Deferred@1949156758 after retry RPC@890294189), errback=(continuation of Deferred@1949156758 after retry RPC@890294189)), so the following Deferred is getting paused: Deferred@1949156758(state=PAUSED, result=Deferred@825187799, callback=org.apache.kudu.client.AsyncKuduClient$6@76cdb7ba -> wakeup thread pool-3-thread-24, errback=passthrough -> wakeup thread pool-3-thread-24) 01:26:51.324 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.324 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.325 [DEBUG - pool-3-thread-76] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.327 [DEBUG - New I/O worker #1005] (Negotiator.java:764) Authenticated connection [id: 0xd8bedd83, /127.1.193.125:50520 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.328 [DEBUG - New I/O worker #1111] (Negotiator.java:764) Authenticated connection [id: 0x0949623e, /127.1.193.126:51259 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.329 [DEBUG - New I/O worker #1043] (Negotiator.java:764) Authenticated connection [id: 0x7e130454, /127.1.193.124:46445 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.329 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.330 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.330 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.330 [DEBUG - New I/O worker #1061] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.331 [DEBUG - New I/O worker #1061] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.333 [DEBUG - Time-limited test] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.333 [DEBUG - New I/O worker #1105] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.333 [DEBUG - New I/O worker #1061] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.333 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.333 [DEBUG - New I/O worker #1210] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.334 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=61), Traces: [0ms] refreshing cache from master, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [60ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:51.335 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.335 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.335 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.336 [DEBUG - New I/O worker #1069] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.336 [DEBUG - New I/O worker #1069] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.336 [DEBUG - New I/O worker #1005] (Deferred.java:1330) callback=retry RPC@850996265 returned Deferred@914800294(state=PENDING, result=null, callback=(continuation of Deferred@1938040338 after retry RPC@850996265), errback=(continuation of Deferred@1938040338 after retry RPC@850996265)), so the following Deferred is getting paused: Deferred@1938040338(state=PAUSED, result=Deferred@914800294, callback=org.apache.kudu.client.AsyncKuduClient$6@399e8dd1 -> wakeup thread pool-3-thread-6, errback=passthrough -> wakeup thread pool-3-thread-6) 01:26:51.337 [DEBUG - pool-3-thread-24] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.337 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.340 [DEBUG - New I/O worker #1069] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.333 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.341 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.341 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.342 [DEBUG - New I/O worker #995] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.342 [DEBUG - New I/O worker #995] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.340 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.337 [DEBUG - New I/O worker #1075] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.337 [DEBUG - New I/O worker #1072] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.342 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.347 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.347 [DEBUG - New I/O worker #1005] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.347 [DEBUG - New I/O worker #1005] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.348 [DEBUG - pool-3-thread-6] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.348 [DEBUG - New I/O worker #1111] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.348 [DEBUG - New I/O worker #1043] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.349 [DEBUG - New I/O worker #1005] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.349 [DEBUG - pool-3-thread-14] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.350 [DEBUG - New I/O worker #1000] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.350 [DEBUG - New I/O worker #995] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.354 [DEBUG - New I/O worker #999] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.356 [DEBUG - pool-3-thread-54] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.373 [DEBUG - pool-3-thread-55] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.382 [DEBUG - New I/O worker #1217] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.382 [DEBUG - New I/O worker #1073] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.383 [DEBUG - New I/O worker #1237] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.387 [DEBUG - pool-3-thread-52] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.393 [DEBUG - New I/O worker #1073] (Negotiator.java:764) Authenticated connection [id: 0xf825a8a0, /127.1.193.125:50528 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.396 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.396 [DEBUG - New I/O worker #1217] (Negotiator.java:764) Authenticated connection [id: 0x404d4f10, /127.1.193.124:46453 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.397 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=40), Traces: [0ms] refreshing cache from master, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [40ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.397 [DEBUG - New I/O worker #1237] (Negotiator.java:764) Authenticated connection [id: 0x97403e12, /127.1.193.126:51267 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.397 [DEBUG - New I/O worker #1073] (Deferred.java:1330) callback=retry RPC@1863145103 returned Deferred@1798492086(state=PENDING, result=null, callback=(continuation of Deferred@774921396 after retry RPC@1863145103), errback=(continuation of Deferred@774921396 after retry RPC@1863145103)), so the following Deferred is getting paused: Deferred@774921396(state=PAUSED, result=Deferred@1798492086, callback=org.apache.kudu.client.AsyncKuduClient$6@1ae83120 -> wakeup thread pool-3-thread-54, errback=passthrough -> wakeup thread pool-3-thread-54) 01:26:51.400 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.404 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.404 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.404 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.404 [DEBUG - New I/O worker #1073] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.405 [DEBUG - New I/O worker #1073] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.406 [DEBUG - pool-3-thread-54] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.406 [DEBUG - New I/O worker #1217] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.406 [DEBUG - New I/O worker #1237] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.406 [DEBUG - New I/O worker #1073] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.407 [DEBUG - pool-3-thread-22] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.409 [DEBUG - pool-3-thread-11] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.417 [DEBUG - New I/O worker #1103] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.417 [DEBUG - New I/O worker #1070] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.419 [DEBUG - New I/O worker #1102] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.426 [DEBUG - New I/O worker #1070] (Negotiator.java:764) Authenticated connection [id: 0x9a27623a, /127.1.193.125:50531 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.428 [DEBUG - New I/O worker #1102] (Negotiator.java:764) Authenticated connection [id: 0x53bca20d, /127.1.193.124:46456 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.430 [DEBUG - New I/O worker #1060] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.430 [DEBUG - New I/O worker #1213] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.431 [DEBUG - New I/O worker #1214] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.432 [DEBUG - New I/O worker #1103] (Negotiator.java:764) Authenticated connection [id: 0x9b1f9015, /127.1.193.126:51270 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.438 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.438 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=112), Traces: [0ms] refreshing cache from master, [89ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [89ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [89ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [106ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [110ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [111ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:51.440 [DEBUG - New I/O worker #1213] (Negotiator.java:764) Authenticated connection [id: 0xcfd13bf4, /127.1.193.124:46459 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.440 [DEBUG - New I/O worker #1214] (Negotiator.java:764) Authenticated connection [id: 0x220bf170, /127.1.193.126:51273 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.440 [DEBUG - New I/O worker #1070] (Deferred.java:1330) callback=retry RPC@1389061233 returned Deferred@1458895600(state=PENDING, result=null, callback=(continuation of Deferred@334917194 after retry RPC@1389061233), errback=(continuation of Deferred@334917194 after retry RPC@1389061233)), so the following Deferred is getting paused: Deferred@334917194(state=PAUSED, result=Deferred@1458895600, callback=org.apache.kudu.client.AsyncKuduClient$6@2b4e6d25 -> wakeup thread pool-3-thread-76, errback=passthrough -> wakeup thread pool-3-thread-76) 01:26:51.445 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.445 [DEBUG - New I/O worker #1023] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.446 [DEBUG - New I/O worker #1124] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.447 [DEBUG - New I/O worker #1285] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.451 [DEBUG - New I/O worker #1209] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.452 [DEBUG - New I/O worker #1055] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.453 [DEBUG - New I/O worker #1050] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.455 [DEBUG - New I/O worker #1023] (Negotiator.java:764) Authenticated connection [id: 0xf7b55e69, /127.1.193.125:50537 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.459 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.460 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=173), Traces: [0ms] refreshing cache from master, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [172ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.461 [DEBUG - New I/O worker #1023] (Deferred.java:1330) callback=retry RPC@1485957671 returned Deferred@1554937325(state=PENDING, result=null, callback=(continuation of Deferred@430759964 after retry RPC@1485957671), errback=(continuation of Deferred@430759964 after retry RPC@1485957671)), so the following Deferred is getting paused: Deferred@430759964(state=PAUSED, result=Deferred@1554937325, callback=org.apache.kudu.client.AsyncKuduClient$6@7fe8a742 -> wakeup thread pool-3-thread-1, errback=passthrough -> wakeup thread pool-3-thread-1) 01:26:51.465 [DEBUG - New I/O worker #1285] (Negotiator.java:764) Authenticated connection [id: 0xe884aa20, /127.1.193.126:51276 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.466 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.466 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.467 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.467 [DEBUG - New I/O worker #1070] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.467 [DEBUG - New I/O worker #1070] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.468 [DEBUG - pool-3-thread-76] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.468 [DEBUG - New I/O worker #1103] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.469 [DEBUG - New I/O worker #1102] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.469 [DEBUG - New I/O worker #1070] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.470 [DEBUG - New I/O worker #1209] (Negotiator.java:764) Authenticated connection [id: 0x6e052b70, /127.1.193.126:51279 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.470 [DEBUG - New I/O worker #1050] (Negotiator.java:764) Authenticated connection [id: 0xc7e586d2, /127.1.193.125:50540 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.467 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.471 [DEBUG - New I/O worker #1060] (Negotiator.java:764) Authenticated connection [id: 0xeba76a76, /127.1.193.125:50534 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.470 [DEBUG - New I/O worker #1124] (Negotiator.java:764) Authenticated connection [id: 0xb1c8f76d, /127.1.193.124:46462 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.472 [DEBUG - New I/O worker #1055] (Negotiator.java:764) Authenticated connection [id: 0xc1b7878a, /127.1.193.124:46465 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.477 [DEBUG - New I/O worker #1018] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.477 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.478 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=191), Traces: [0ms] refreshing cache from master, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [188ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [191ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.479 [DEBUG - New I/O worker #1050] (Deferred.java:1330) callback=retry RPC@1374058117 returned Deferred@1440344161(state=PENDING, result=null, callback=(continuation of Deferred@282819774 after retry RPC@1374058117), errback=(continuation of Deferred@282819774 after retry RPC@1374058117)), so the following Deferred is getting paused: Deferred@282819774(state=PAUSED, result=Deferred@1440344161, callback=org.apache.kudu.client.AsyncKuduClient$6@43b15cbf -> wakeup thread pool-3-thread-3, errback=passthrough -> wakeup thread pool-3-thread-3) 01:26:51.479 [DEBUG - New I/O worker #1016] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.481 [DEBUG - New I/O worker #1019] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.481 [DEBUG - New I/O worker #1204] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.485 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.486 [DEBUG - New I/O worker #1038] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.489 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.489 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.490 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.490 [DEBUG - New I/O worker #1050] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.490 [DEBUG - New I/O worker #1204] (Negotiator.java:764) Authenticated connection [id: 0x373c52fd, /127.1.193.126:51285 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.490 [DEBUG - New I/O worker #1050] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.491 [DEBUG - New I/O worker #963] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.492 [DEBUG - pool-3-thread-3] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.492 [DEBUG - New I/O worker #1016] (Negotiator.java:764) Authenticated connection [id: 0x17ea18b1, /127.1.193.125:50543 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.492 [DEBUG - New I/O worker #1209] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.495 [DEBUG - New I/O worker #1055] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.495 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.495 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.496 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.496 [DEBUG - New I/O worker #1023] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.496 [DEBUG - New I/O worker #1023] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.497 [DEBUG - pool-3-thread-1] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.497 [DEBUG - New I/O worker #1124] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.498 [DEBUG - New I/O worker #1114] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.498 [DEBUG - New I/O worker #1285] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.499 [DEBUG - New I/O worker #1040] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.501 [DEBUG - New I/O worker #988] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.495 [DEBUG - New I/O worker #1050] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.501 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.498 [DEBUG - New I/O worker #1023] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.503 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=200), Traces: [0ms] refreshing cache from master, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [142ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [153ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [199ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.504 [DEBUG - New I/O worker #1060] (Deferred.java:1330) callback=retry RPC@2048261350 returned Deferred@2116708220(state=PENDING, result=null, callback=(continuation of Deferred@992540381 after retry RPC@2048261350), errback=(continuation of Deferred@992540381 after retry RPC@2048261350)), so the following Deferred is getting paused: Deferred@992540381(state=PAUSED, result=Deferred@2116708220, callback=org.apache.kudu.client.AsyncKuduClient$6@5f7e3fd8 -> wakeup thread pool-3-thread-63, errback=passthrough -> wakeup thread pool-3-thread-63) 01:26:51.506 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.508 [DEBUG - New I/O worker #1040] (Negotiator.java:764) Authenticated connection [id: 0xc7573d78, /127.1.193.124:46474 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.510 [DEBUG - New I/O worker #988] (Negotiator.java:764) Authenticated connection [id: 0x6ca82799, /127.1.193.125:50549 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.511 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.511 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=226), Traces: [0ms] refreshing cache from master, [188ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [188ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [188ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.512 [DEBUG - New I/O worker #1016] (Deferred.java:1330) callback=retry RPC@664903648 returned Deferred@597561292(state=PENDING, result=null, callback=(continuation of Deferred@1721538011 after retry RPC@664903648), errback=(continuation of Deferred@1721538011 after retry RPC@664903648)), so the following Deferred is getting paused: Deferred@1721538011(state=PAUSED, result=Deferred@597561292, callback=org.apache.kudu.client.AsyncKuduClient$6@7cc54e30 -> wakeup thread pool-3-thread-15, errback=passthrough -> wakeup thread pool-3-thread-15) 01:26:51.515 [DEBUG - New I/O worker #963] (Negotiator.java:764) Authenticated connection [id: 0xcd008c54, /127.1.193.125:50546 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.515 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.517 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.517 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.518 [DEBUG - New I/O worker #1038] (Negotiator.java:764) Authenticated connection [id: 0xf4fcfbc1, /127.1.193.124:46471 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.518 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.518 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.519 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=237), Traces: [0ms] refreshing cache from master, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [216ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [237ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.519 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.520 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=243), Traces: [0ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [237ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.518 [DEBUG - New I/O worker #1060] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.521 [DEBUG - New I/O worker #1060] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.521 [DEBUG - pool-3-thread-63] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.522 [DEBUG - New I/O worker #1213] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.522 [DEBUG - New I/O worker #1214] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.523 [DEBUG - New I/O worker #1060] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.525 [DEBUG - New I/O worker #1241] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.520 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.519 [DEBUG - New I/O worker #963] (Deferred.java:1330) callback=retry RPC@608891279 returned Deferred@544567774(state=PENDING, result=null, callback=(continuation of Deferred@1702362036 after retry RPC@608891279), errback=(continuation of Deferred@1702362036 after retry RPC@608891279)), so the following Deferred is getting paused: Deferred@1702362036(state=PAUSED, result=Deferred@544567774, callback=org.apache.kudu.client.AsyncKuduClient$6@27edc5de -> wakeup thread pool-3-thread-59, errback=passthrough -> wakeup thread pool-3-thread-59) 01:26:51.527 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.526 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.525 [DEBUG - New I/O worker #1101] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.530 [DEBUG - pool-3-thread-42] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.521 [DEBUG - New I/O worker #988] (Deferred.java:1330) callback=retry RPC@401364659 returned Deferred@332644502(state=PENDING, result=null, callback=(continuation of Deferred@1456561288 after retry RPC@401364659), errback=(continuation of Deferred@1456561288 after retry RPC@401364659)), so the following Deferred is getting paused: Deferred@1456561288(state=PAUSED, result=Deferred@332644502, callback=org.apache.kudu.client.AsyncKuduClient$6@12e1a92f -> wakeup thread pool-3-thread-13, errback=passthrough -> wakeup thread pool-3-thread-13) 01:26:51.527 [DEBUG - New I/O worker #1071] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.536 [DEBUG - New I/O worker #1114] (Negotiator.java:764) Authenticated connection [id: 0x9342cbf9, /127.1.193.126:51288 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.539 [DEBUG - New I/O worker #1241] (Negotiator.java:764) Authenticated connection [id: 0x747d739a, /127.1.193.126:51291 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.540 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.540 [DEBUG - New I/O worker #1016] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.543 [DEBUG - New I/O worker #1016] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.543 [DEBUG - New I/O worker #1018] (Negotiator.java:764) Authenticated connection [id: 0xd75ef9ac, /127.1.193.124:46468 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.531 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.544 [DEBUG - New I/O worker #1101] (Negotiator.java:764) Authenticated connection [id: 0x0f16f4f2, /127.1.193.124:46477 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.544 [DEBUG - pool-3-thread-15] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.544 [DEBUG - New I/O worker #1016] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.544 [DEBUG - New I/O worker #1019] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: connection disconnected 01:26:51.545 [INFO - New I/O worker #1019] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: connection disconnected 01:26:51.544 [DEBUG - New I/O worker #1018] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.546 [INFO - New I/O worker #1018] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:51.548 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.548 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.548 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.549 [DEBUG - New I/O worker #963] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.549 [DEBUG - New I/O worker #963] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.549 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.550 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.550 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.551 [DEBUG - New I/O worker #988] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.551 [DEBUG - New I/O worker #988] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.552 [DEBUG - pool-3-thread-59] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.553 [DEBUG - New I/O worker #1038] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.553 [DEBUG - pool-3-thread-13] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.553 [DEBUG - New I/O worker #1040] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.554 [DEBUG - New I/O worker #1114] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.550 [DEBUG - New I/O worker #1071] (Negotiator.java:764) Authenticated connection [id: 0x3d252f18, /127.1.193.125:50552 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.553 [DEBUG - New I/O worker #963] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.553 [DEBUG - New I/O worker #1204] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.551954 1951 connection.cc:554] Connection torn down before Call kudu.master.MasterService.ConnectToMaster from 127.1.193.124:46468 (request call id 0) could send its response 01:26:51.555 [DEBUG - New I/O worker #988] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.557 [DEBUG - pool-3-thread-46] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.561 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:51.561542 4571 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.475975 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51282 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.479540 (+ 3565us) server_negotiation.cc:184] Beginning negotiation 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.479569 (+ 29us) server_negotiation.cc:373] Waiting for connection header 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.479617 (+ 48us) server_negotiation.cc:381] Connection header received 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.480277 (+ 660us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.480312 (+ 35us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.480472 (+ 160us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.482944 (+ 2472us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.483232 (+ 288us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.492794 (+ 9562us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.494957 (+ 2163us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.538976 (+ 44019us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.544924 (+ 5948us) server_negotiation.cc:584] Negotiated auth-only TLSv1 with cipher AES256-SHA SSLv3 Kx=RSA Au=RSA Enc=AES(256) Mac=SHA1 01:26:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:51.549293 (+ 4369us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51282: BlockingRecv error: recv got EOF from 0.0.0.0:0 (error 108) 01:26:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"mutex_wait_us":7702,"server-negotiator.queue_time_us":3331} 01:26:51.561 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=286), Traces: [0ms] refreshing cache from master, [246ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [246ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [247ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [268ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [278ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.564 [DEBUG - New I/O worker #1071] (Deferred.java:1330) callback=retry RPC@1940763971 returned Deferred@2006073326(state=PENDING, result=null, callback=(continuation of Deferred@848347000 after retry RPC@1940763971), errback=(continuation of Deferred@848347000 after retry RPC@1940763971)), so the following Deferred is getting paused: Deferred@848347000(state=PAUSED, result=Deferred@2006073326, callback=org.apache.kudu.client.AsyncKuduClient$6@1df1d310 -> wakeup thread pool-3-thread-28, errback=passthrough -> wakeup thread pool-3-thread-28) 01:26:51.568 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.575 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.576 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.576 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.576 [DEBUG - New I/O worker #1071] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.577 [DEBUG - New I/O worker #1071] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.578 [DEBUG - pool-3-thread-28] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.579 [DEBUG - New I/O worker #1101] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.579 [DEBUG - New I/O worker #1071] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.579 [DEBUG - New I/O worker #1241] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.654 [INFO - New I/O worker #1172] (Connection.java:435) [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:51.718 [DEBUG - New I/O worker #1172] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:51.763 [INFO - New I/O worker #1172] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:51.695 [DEBUG - pool-3-thread-41] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.687 [DEBUG - pool-3-thread-44] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.779 [INFO - New I/O worker #1292] (Connection.java:435) [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:51.781 [DEBUG - New I/O worker #1292] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:51.782 [INFO - New I/O worker #1292] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:51.801 [DEBUG - New I/O worker #1146] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.800 [DEBUG - New I/O worker #1163] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.799 [DEBUG - New I/O worker #977] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.799 [DEBUG - New I/O worker #1310] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.792 [DEBUG - New I/O worker #976] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.798 [DEBUG - New I/O worker #982] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.817 [DEBUG - pool-3-thread-38] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.836 [DEBUG - New I/O worker #977] (Negotiator.java:764) Authenticated connection [id: 0xb0ba08ec, /127.1.193.124:46484 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.839 [DEBUG - New I/O worker #982] (Negotiator.java:764) Authenticated connection [id: 0xf6b5a142, /127.1.193.125:50557 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.843 [DEBUG - New I/O worker #1163] (Negotiator.java:764) Authenticated connection [id: 0xcffc844f, /127.1.193.126:51298 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.846 [DEBUG - New I/O worker #1310] (Negotiator.java:764) Authenticated connection [id: 0xaba33e6a, /127.1.193.126:51296 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.850 [DEBUG - New I/O worker #1146] (Negotiator.java:764) Authenticated connection [id: 0xa8a34da3, /127.1.193.124:46482 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.851 [DEBUG - New I/O worker #1271] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.853 [DEBUG - New I/O worker #1193] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.853 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.854 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=614), Traces: [0ms] refreshing cache from master, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [614ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.855 [DEBUG - New I/O worker #1030] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.856 [DEBUG - New I/O worker #982] (Deferred.java:1330) callback=retry RPC@454326034 returned Deferred@522973352(state=PENDING, result=null, callback=(continuation of Deferred@1512668457 after retry RPC@454326034), errback=(continuation of Deferred@1512668457 after retry RPC@454326034)), so the following Deferred is getting paused: Deferred@1512668457(state=PAUSED, result=Deferred@522973352, callback=org.apache.kudu.client.AsyncKuduClient$6@532d20ef -> wakeup thread pool-3-thread-4, errback=passthrough -> wakeup thread pool-3-thread-4) 01:26:51.857 [DEBUG - New I/O worker #976] (Negotiator.java:764) Authenticated connection [id: 0x7b5131f3, /127.1.193.125:50556 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.863 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.863 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=614), Traces: [0ms] refreshing cache from master, [338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [510ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [599ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [606ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [613ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.864 [DEBUG - New I/O worker #976] (Deferred.java:1330) callback=retry RPC@2074354442 returned Deferred@2140911782(state=PENDING, result=null, callback=(continuation of Deferred@983116081 after retry RPC@2074354442), errback=(continuation of Deferred@983116081 after retry RPC@2074354442)), so the following Deferred is getting paused: Deferred@983116081(state=PAUSED, result=Deferred@2140911782, callback=org.apache.kudu.client.AsyncKuduClient$6@510fa0d7 -> wakeup thread pool-3-thread-67, errback=passthrough -> wakeup thread pool-3-thread-67) 01:26:51.867 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.873 [DEBUG - New I/O worker #1203] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.873 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.874 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.875 [DEBUG - New I/O worker #1039] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.875 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.876 [DEBUG - New I/O worker #976] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.878 [DEBUG - New I/O worker #976] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.878 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.877 [DEBUG - New I/O worker #965] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.882 [DEBUG - New I/O worker #976] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.880 [DEBUG - New I/O worker #1163] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.883 [DEBUG - New I/O worker #1203] (Negotiator.java:764) Authenticated connection [id: 0x37420a83, /127.1.193.126:51304 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.880 [DEBUG - New I/O worker #977] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.880 [DEBUG - pool-3-thread-67] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.880 [DEBUG - New I/O worker #1030] (Negotiator.java:764) Authenticated connection [id: 0x86684b42, /127.1.193.125:50562 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.879 [DEBUG - New I/O worker #1271] (Negotiator.java:764) Authenticated connection [id: 0x6b7ebce3, /127.1.193.126:51301 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.883 [DEBUG - New I/O worker #1193] (Negotiator.java:764) Authenticated connection [id: 0xd8ce55be, /127.1.193.124:46487 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.891 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.892 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=333), Traces: [0ms] refreshing cache from master, [287ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [288ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [288ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [330ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [333ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:51.892 [DEBUG - New I/O worker #1030] (Deferred.java:1330) callback=retry RPC@1863174929 returned Deferred@1798459944(state=PENDING, result=null, callback=(continuation of Deferred@774947114 after retry RPC@1863174929), errback=(continuation of Deferred@774947114 after retry RPC@1863174929)), so the following Deferred is getting paused: Deferred@774947114(state=PAUSED, result=Deferred@1798459944, callback=org.apache.kudu.client.AsyncKuduClient$6@1e002d4d -> wakeup thread pool-3-thread-46, errback=passthrough -> wakeup thread pool-3-thread-46) 01:26:51.893 [DEBUG - New I/O worker #965] (Negotiator.java:764) Authenticated connection [id: 0x1d5da3fe, /127.1.193.125:50565 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.894 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.894 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.894 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.894 [DEBUG - New I/O worker #982] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.895 [DEBUG - New I/O worker #982] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.896 [DEBUG - pool-3-thread-4] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.896 [DEBUG - New I/O worker #1310] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.896 [DEBUG - New I/O worker #1146] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.896 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.898 [DEBUG - New I/O worker #1039] (Negotiator.java:764) Authenticated connection [id: 0x941d32dd, /127.1.193.124:46490 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.899 [DEBUG - New I/O worker #982] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.901 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.902 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.902 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.901 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.902 [DEBUG - New I/O worker #1030] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.903 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=371), Traces: [0ms] refreshing cache from master, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [363ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [369ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [370ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.903 [DEBUG - New I/O worker #1030] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.903 [DEBUG - New I/O worker #965] (Deferred.java:1330) callback=retry RPC@2043428731 returned Deferred@2113123482(state=PENDING, result=null, callback=(continuation of Deferred@955331904 after retry RPC@2043428731), errback=(continuation of Deferred@955331904 after retry RPC@2043428731)), so the following Deferred is getting paused: Deferred@955331904(state=PAUSED, result=Deferred@2113123482, callback=org.apache.kudu.client.AsyncKuduClient$6@11ac493b -> wakeup thread pool-3-thread-42, errback=passthrough -> wakeup thread pool-3-thread-42) 01:26:51.903 [DEBUG - New I/O worker #1030] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.904 [DEBUG - New I/O worker #1193] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.904 [DEBUG - New I/O worker #1271] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.903 [DEBUG - pool-3-thread-46] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.909 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.912 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.913 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.913 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.913 [DEBUG - New I/O worker #965] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.913 [DEBUG - New I/O worker #965] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.914 [DEBUG - New I/O worker #965] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.914 [DEBUG - pool-3-thread-42] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.915 [DEBUG - New I/O worker #1203] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.914 [DEBUG - New I/O worker #1039] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.929 [DEBUG - New I/O worker #1034] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.929 [DEBUG - New I/O worker #1270] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.930 [DEBUG - New I/O worker #1195] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.939 [DEBUG - New I/O worker #1270] (Negotiator.java:764) Authenticated connection [id: 0xf63f3553, /127.1.193.126:51307 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.939 [DEBUG - New I/O worker #1195] (Negotiator.java:764) Authenticated connection [id: 0x5cda27eb, /127.1.193.124:46493 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.941 [DEBUG - New I/O worker #1034] (Negotiator.java:764) Authenticated connection [id: 0x27d5bf3f, /127.1.193.125:50568 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.945 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.946 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=536), Traces: [0ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [534ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [535ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [536ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.949 [DEBUG - New I/O worker #1034] (Deferred.java:1330) callback=retry RPC@238547395 returned Deferred@168326365(state=PENDING, result=null, callback=(continuation of Deferred@1326119928 after retry RPC@238547395), errback=(continuation of Deferred@1326119928 after retry RPC@238547395)), so the following Deferred is getting paused: Deferred@1326119928(state=PAUSED, result=Deferred@168326365, callback=org.apache.kudu.client.AsyncKuduClient$6@4f0788c3 -> wakeup thread pool-3-thread-11, errback=passthrough -> wakeup thread pool-3-thread-11) 01:26:51.949 [DEBUG - New I/O worker #1251] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.950 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.951 [DEBUG - New I/O worker #1056] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.956 [DEBUG - pool-3-thread-79] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:51.957 [DEBUG - New I/O worker #1251] (Negotiator.java:764) Authenticated connection [id: 0x0ff981ad, /127.1.193.126:51310 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.958 [DEBUG - New I/O worker #1208] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.965 [DEBUG - New I/O worker #1208] (Negotiator.java:764) Authenticated connection [id: 0xd5862039, /127.1.193.124:46496 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.966 [DEBUG - New I/O worker #972] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:51.966 [DEBUG - New I/O worker #1044] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:51.967 [DEBUG - New I/O worker #1113] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:51.970 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.970 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.970 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.973 [DEBUG - New I/O worker #1034] (NetUtil.java:106) Resolved IP of `127.1.193.66' to /127.1.193.66 in 2131187ns 01:26:51.973 [DEBUG - New I/O worker #1034] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.973 [DEBUG - New I/O worker #1034] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.974 [DEBUG - New I/O worker #972] (Negotiator.java:764) Authenticated connection [id: 0x6a659db3, /127.1.193.125:50574 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.976 [DEBUG - New I/O worker #1044] (Negotiator.java:764) Authenticated connection [id: 0xaf2de2fb, /127.1.193.124:46499 => /127.1.193.124:58247] using SASL/PLAIN 01:26:51.976 [DEBUG - New I/O worker #1113] (Negotiator.java:764) Authenticated connection [id: 0x7735aa3e, /127.1.193.126:51313 => /127.1.193.126:40739] using SASL/PLAIN 01:26:51.977 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.976 [DEBUG - New I/O worker #1056] (Negotiator.java:764) Authenticated connection [id: 0xc7d00735, /127.1.193.125:50571 => /127.1.193.125:37435] using SASL/PLAIN 01:26:51.977 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=589), Traces: [0ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [589ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.978 [DEBUG - New I/O worker #972] (Deferred.java:1330) callback=retry RPC@1776527851 returned Deferred@1843142671(state=PENDING, result=null, callback=(continuation of Deferred@685679568 after retry RPC@1776527851), errback=(continuation of Deferred@685679568 after retry RPC@1776527851)), so the following Deferred is getting paused: Deferred@685679568(state=PAUSED, result=Deferred@1843142671, callback=org.apache.kudu.client.AsyncKuduClient$6@6d646a07 -> wakeup thread pool-3-thread-52, errback=passthrough -> wakeup thread pool-3-thread-52) 01:26:51.980 [DEBUG - New I/O worker #1034] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.979 [DEBUG - New I/O worker #1195] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.979 [DEBUG - pool-3-thread-11] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.980 [DEBUG - New I/O worker #1270] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.981 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.981 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:51.982 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=574), Traces: [0ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [553ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [573ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:51.982 [DEBUG - New I/O worker #1056] (Deferred.java:1330) callback=retry RPC@1142565036 returned Deferred@1076212983(state=PENDING, result=null, callback=(continuation of Deferred@86450839 after retry RPC@1142565036), errback=(continuation of Deferred@86450839 after retry RPC@1142565036)), so the following Deferred is getting paused: Deferred@86450839(state=PAUSED, result=Deferred@1076212983, callback=org.apache.kudu.client.AsyncKuduClient$6@21a2f453 -> wakeup thread pool-3-thread-22, errback=passthrough -> wakeup thread pool-3-thread-22) 01:26:51.984 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:51.990 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.991 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.991 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.991 [DEBUG - New I/O worker #1056] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.992 [DEBUG - New I/O worker #1056] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.991 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:51.993 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:51.993 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:51.993 [DEBUG - New I/O worker #972] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:51.992 [DEBUG - New I/O worker #1208] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.992 [DEBUG - New I/O worker #1056] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:51.992 [DEBUG - pool-3-thread-22] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.994 [DEBUG - New I/O worker #972] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:51.993 [DEBUG - New I/O worker #1251] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.996 [DEBUG - pool-3-thread-52] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:51.996 [DEBUG - New I/O worker #1044] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:51.996 [DEBUG - New I/O worker #1113] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:51.997 [DEBUG - New I/O worker #972] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.003 [DEBUG - New I/O worker #1021] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.004 [DEBUG - New I/O worker #1194] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.008 [DEBUG - New I/O worker #1196] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.009 [DEBUG - New I/O worker #1021] (Negotiator.java:764) Authenticated connection [id: 0xb73e9977, /127.1.193.125:50577 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.012 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.013 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=639), Traces: [0ms] refreshing cache from master, [626ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [626ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [627ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.016 [DEBUG - New I/O worker #1021] (Deferred.java:1330) callback=retry RPC@363746789 returned Deferred@294769544(state=PENDING, result=null, callback=(continuation of Deferred@1418944478 after retry RPC@363746789), errback=(continuation of Deferred@1418944478 after retry RPC@363746789)), so the following Deferred is getting paused: Deferred@1418944478(state=PAUSED, result=Deferred@294769544, callback=org.apache.kudu.client.AsyncKuduClient$6@2eb83cc -> wakeup thread pool-3-thread-55, errback=passthrough -> wakeup thread pool-3-thread-55) 01:26:52.017 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.020 [DEBUG - New I/O worker #1196] (Negotiator.java:764) Authenticated connection [id: 0x67cb21c6, /127.1.193.126:51316 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.021 [DEBUG - New I/O worker #1194] (Negotiator.java:764) Authenticated connection [id: 0x3a365082, /127.1.193.124:46502 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.021 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.023 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.024 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.024 [DEBUG - New I/O worker #1021] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.025 [DEBUG - New I/O worker #1021] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.025 [DEBUG - pool-3-thread-55] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.026 [DEBUG - New I/O worker #1196] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.026 [INFO - New I/O worker #1196] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: connection disconnected 01:26:52.025 [DEBUG - New I/O worker #1194] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:52.026033 1815 connection.cc:554] Connection torn down before Call kudu.master.MasterService.ConnectToMaster from 127.1.193.126:51316 (request call id 0) could send its response 01:26:52.026 [DEBUG - New I/O worker #1021] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.031 [DEBUG - pool-3-thread-51] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.042 [DEBUG - New I/O worker #1274] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.042 [DEBUG - New I/O worker #1189] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.042 [DEBUG - New I/O worker #1027] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.051 [DEBUG - New I/O worker #1274] (Negotiator.java:764) Authenticated connection [id: 0x01fdbdac, /127.1.193.126:51319 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.057 [DEBUG - New I/O worker #1027] (Negotiator.java:764) Authenticated connection [id: 0xb490fded, /127.1.193.125:50580 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.059 [DEBUG - New I/O worker #1189] (Negotiator.java:764) Authenticated connection [id: 0xbcc9b772, /127.1.193.124:46505 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.061 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.062 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=104), Traces: [0ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [98ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [105ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:52.063 [DEBUG - New I/O worker #1027] (Deferred.java:1330) callback=retry RPC@1676745603 returned Deferred@1741591333(state=PENDING, result=null, callback=(continuation of Deferred@583799224 after retry RPC@1676745603), errback=(continuation of Deferred@583799224 after retry RPC@1676745603)), so the following Deferred is getting paused: Deferred@583799224(state=PAUSED, result=Deferred@1741591333, callback=org.apache.kudu.client.AsyncKuduClient$6@7862b1e4 -> wakeup thread pool-3-thread-79, errback=passthrough -> wakeup thread pool-3-thread-79) 01:26:52.065 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.068 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.068 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.069 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.069 [DEBUG - New I/O worker #1027] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.069 [DEBUG - New I/O worker #1027] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.070 [DEBUG - pool-3-thread-79] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.070 [DEBUG - New I/O worker #1189] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.070 [DEBUG - New I/O worker #1274] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.071 [DEBUG - New I/O worker #1027] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.095 [DEBUG - pool-3-thread-7] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.100 [DEBUG - pool-3-thread-68] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.144 [DEBUG - pool-3-thread-47] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.169 [DEBUG - New I/O worker #1015] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.182 [DEBUG - New I/O worker #1015] (Negotiator.java:764) Authenticated connection [id: 0x6dcc65ae, /127.1.193.125:50583 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.184 [DEBUG - New I/O worker #1290] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.185 [DEBUG - New I/O worker #1287] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.186 [DEBUG - New I/O worker #1128] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.187 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.187 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=369), Traces: [0ms] refreshing cache from master, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [346ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [368ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.188 [DEBUG - New I/O worker #1015] (Deferred.java:1330) callback=retry RPC@1336106628 returned Deferred@1268569000(state=PENDING, result=null, callback=(continuation of Deferred@245257407 after retry RPC@1336106628), errback=(continuation of Deferred@245257407 after retry RPC@1336106628)), so the following Deferred is getting paused: Deferred@245257407(state=PAUSED, result=Deferred@1268569000, callback=org.apache.kudu.client.AsyncKuduClient$6@6ad88fcc -> wakeup thread pool-3-thread-38, errback=passthrough -> wakeup thread pool-3-thread-38) 01:26:52.188 [DEBUG - New I/O worker #1051] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.188 [DEBUG - New I/O worker #1126] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.191 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.200 [DEBUG - New I/O worker #1287] (Negotiator.java:764) Authenticated connection [id: 0xafb7058f, /127.1.193.126:51322 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.200 [DEBUG - New I/O worker #1290] (Negotiator.java:764) Authenticated connection [id: 0x233d6d29, /127.1.193.126:51325 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.191 [DEBUG - New I/O worker #1008] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.203 [DEBUG - New I/O worker #1051] (Negotiator.java:764) Authenticated connection [id: 0xeab642a0, /127.1.193.125:50589 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.203 [DEBUG - New I/O worker #1109] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.205 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.205 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.206 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.206 [DEBUG - New I/O worker #1015] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.207 [DEBUG - New I/O worker #1015] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.208 [DEBUG - pool-3-thread-38] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.208 [DEBUG - New I/O worker #1015] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.210 [DEBUG - New I/O worker #1008] (Negotiator.java:764) Authenticated connection [id: 0x38338019, /127.1.193.125:50586 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.211 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.212 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=445), Traces: [0ms] refreshing cache from master, [408ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [445ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.198 [DEBUG - New I/O worker #1126] (Negotiator.java:764) Authenticated connection [id: 0x642c2a2d, /127.1.193.124:46508 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.212 [DEBUG - New I/O worker #1051] (Deferred.java:1330) callback=retry RPC@983689447 returned Deferred@1050569579(state=PENDING, result=null, callback=(continuation of Deferred@2073879260 after retry RPC@983689447), errback=(continuation of Deferred@2073879260 after retry RPC@983689447)), so the following Deferred is getting paused: Deferred@2073879260(state=PAUSED, result=Deferred@1050569579, callback=org.apache.kudu.client.AsyncKuduClient$6@7f7e536c -> wakeup thread pool-3-thread-44, errback=passthrough -> wakeup thread pool-3-thread-44) 01:26:52.212 [DEBUG - New I/O worker #2] (Connection.java:429) [peer master-127.1.193.124:58247(127.1.193.124:58247)] encountered a read timeout; closing the channel 01:26:52.212 [DEBUG - New I/O worker #2] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] encountered a read timeout; closing the channel 01:26:52.198 [DEBUG - New I/O worker #1128] (Negotiator.java:764) Authenticated connection [id: 0x78b40c7b, /127.1.193.124:46511 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.212 [DEBUG - New I/O worker #3] (Connection.java:429) [peer master-127.1.193.126:40739(127.1.193.126:40739)] encountered a read timeout; closing the channel 01:26:52.213 [DEBUG - New I/O worker #3] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: [peer master-127.1.193.126:40739(127.1.193.126:40739)] encountered a read timeout; closing the channel 01:26:52.216 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.216 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=450), Traces: [0ms] refreshing cache from master, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [409ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [440ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [450ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.216 [DEBUG - New I/O worker #1008] (Deferred.java:1330) callback=retry RPC@436885111 returned Deferred@506843630(state=PENDING, result=null, callback=(continuation of Deferred@1530354764 after retry RPC@436885111), errback=(continuation of Deferred@1530354764 after retry RPC@436885111)), so the following Deferred is getting paused: Deferred@1530354764(state=PAUSED, result=Deferred@506843630, callback=org.apache.kudu.client.AsyncKuduClient$6@22d704fe -> wakeup thread pool-3-thread-41, errback=passthrough -> wakeup thread pool-3-thread-41) 01:26:52.212 [INFO - New I/O worker #1126] (Connection.java:435) [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.217 [DEBUG - New I/O worker #1126] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.218 [INFO - New I/O worker #1126] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.208 [DEBUG - New I/O worker #1287] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.205 [DEBUG - New I/O worker #1205] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.222 [DEBUG - New I/O worker #1672] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.218 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.218 [DEBUG - New I/O worker #1109] (Negotiator.java:764) Authenticated connection [id: 0x3b9876a3, /127.1.193.124:46514 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.217 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.224 [DEBUG - New I/O worker #1679] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.288 [DEBUG - New I/O worker #970] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.290 [DEBUG - New I/O worker #1679] (Negotiator.java:764) Authenticated connection [id: 0xa984031f, /127.1.193.126:51331 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.295 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.296 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.296 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.296 [DEBUG - New I/O worker #1008] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.296 [DEBUG - New I/O worker #1008] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.297 [DEBUG - New I/O worker #1128] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.298 [DEBUG - New I/O worker #1290] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.298 [DEBUG - New I/O worker #1008] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.298 [DEBUG - New I/O worker #1677] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.298 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.300 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.300 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.300 [DEBUG - New I/O worker #1051] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.301 [DEBUG - New I/O worker #1051] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.304 [DEBUG - pool-3-thread-21] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.305 [DEBUG - New I/O worker #1042] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.305 [DEBUG - New I/O worker #1599] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.305 [DEBUG - New I/O worker #1623] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.306 [DEBUG - New I/O worker #1200] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.308 [DEBUG - New I/O worker #1672] (Negotiator.java:764) Authenticated connection [id: 0xe86cdfec, /127.1.193.125:50592 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.309 [DEBUG - pool-3-thread-44] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.309 [INFO - New I/O worker #1205] (Connection.java:435) [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:52.309 [DEBUG - New I/O worker #1205] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:52.309 [DEBUG - New I/O worker #1617] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.310 [DEBUG - pool-3-thread-41] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.309 [DEBUG - New I/O worker #1109] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.309 [DEBUG - New I/O worker #1051] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.309 [INFO - New I/O worker #1205] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:52.312 [DEBUG - New I/O worker #1205] (Negotiator.java:764) Authenticated connection [id: 0x0ee4176e, /127.1.193.126:51328 :> /127.1.193.126:40739] using SASL/PLAIN 01:26:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:52.311976 4812 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.176485 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51328 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.190026 (+ 13541us) server_negotiation.cc:184] Beginning negotiation 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.190055 (+ 29us) server_negotiation.cc:373] Waiting for connection header 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.190099 (+ 44us) server_negotiation.cc:381] Connection header received 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.190211 (+ 112us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.190237 (+ 26us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.190372 (+ 135us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.205219 (+ 14847us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.220931 (+ 15712us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.221517 (+ 586us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.222958 (+ 1441us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.224149 (+ 1191us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.224400 (+ 251us) server_negotiation.cc:584] Negotiated auth-only TLSv1 with cipher AES256-SHA SSLv3 Kx=RSA Au=RSA Enc=AES(256) Mac=SHA1 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226127 (+ 1727us) server_negotiation.cc:337] Received SASL_INITIATE NegotiatePB request 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226161 (+ 34us) server_negotiation.cc:746] Received SASL_INITIATE request from client 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226187 (+ 26us) server_negotiation.cc:757] Client requested to use mechanism: PLAIN 01:26:52.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226247 (+ 60us) server_negotiation.cc:786] Calling sasl_server_start() 01:26:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226318 (+ 71us) server_negotiation.cc:945] Received PLAIN auth, user=jenkins 01:26:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226388 (+ 70us) server_negotiation.cc:349] Sending SASL_SUCCESS NegotiatePB response 01:26:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.226526 (+ 138us) server_negotiation.cc:890] Waiting for connection context 01:26:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.310324 (+ 83798us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51328: BlockingRecv error: recv got EOF from 127.1.193.126:51328 (error 108) 01:26:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":13270,"thread_start_us":22043,"threads_started":1} 01:26:52.315 [DEBUG - New I/O worker #970] (Negotiator.java:764) Authenticated connection [id: 0x08134d16, /127.1.193.125:50595 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.317 [DEBUG - New I/O worker #1677] (Negotiator.java:764) Authenticated connection [id: 0x58408f53, /127.1.193.124:46517 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.319 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.319 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=288), Traces: [0ms] refreshing cache from master, [253ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [287ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.320 [DEBUG - New I/O worker #970] (Deferred.java:1330) callback=retry RPC@1675975997 returned Deferred@1742394777(state=PENDING, result=null, callback=(continuation of Deferred@584603398 after retry RPC@1675975997), errback=(continuation of Deferred@584603398 after retry RPC@1675975997)), so the following Deferred is getting paused: Deferred@584603398(state=PAUSED, result=Deferred@1742394777, callback=org.apache.kudu.client.AsyncKuduClient$6@5af57042 -> wakeup thread pool-3-thread-51, errback=passthrough -> wakeup thread pool-3-thread-51) 01:26:52.320 [DEBUG - New I/O worker #1200] (Negotiator.java:764) Authenticated connection [id: 0x72ce7228, /127.1.193.126:51334 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.321 [DEBUG - New I/O worker #1042] (Negotiator.java:764) Authenticated connection [id: 0x66b1f8cd, /127.1.193.124:46520 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.322 [DEBUG - New I/O worker #1617] (Negotiator.java:764) Authenticated connection [id: 0xc15590ed, /127.1.193.125:50601 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.323 [DEBUG - New I/O worker #1623] (Negotiator.java:764) Authenticated connection [id: 0x4c8acb24, /127.1.193.126:51340 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.324 [DEBUG - New I/O worker #1620] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.326 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.327 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=182), Traces: [0ms] refreshing cache from master, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [74ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [151ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [177ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.327 [DEBUG - New I/O worker #1672] (Deferred.java:1330) callback=retry RPC@1686860800 returned Deferred@1622473801(state=PENDING, result=null, callback=(continuation of Deferred@632716859 after retry RPC@1686860800), errback=(continuation of Deferred@632716859 after retry RPC@1686860800)), so the following Deferred is getting paused: Deferred@632716859(state=PAUSED, result=Deferred@1622473801, callback=org.apache.kudu.client.AsyncKuduClient$6@2bcdd0eb -> wakeup thread pool-3-thread-47, errback=passthrough -> wakeup thread pool-3-thread-47) 01:26:52.328 [DEBUG - New I/O worker #1601] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.332 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.332 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=231), Traces: [0ms] refreshing cache from master, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [231ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.332 [DEBUG - New I/O worker #1617] (Deferred.java:1330) callback=retry RPC@275554323 returned Deferred@340997830(state=PENDING, result=null, callback=(continuation of Deferred@1364306472 after retry RPC@275554323), errback=(continuation of Deferred@1364306472 after retry RPC@275554323)), so the following Deferred is getting paused: Deferred@1364306472(state=PAUSED, result=Deferred@340997830, callback=org.apache.kudu.client.AsyncKuduClient$6@15531b59 -> wakeup thread pool-3-thread-68, errback=passthrough -> wakeup thread pool-3-thread-68) 01:26:52.332 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.333 [DEBUG - New I/O worker #1599] (Negotiator.java:764) Authenticated connection [id: 0x914035c1, /127.1.193.124:46523 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.336 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.337 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.337 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.337 [DEBUG - New I/O worker #970] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.338 [DEBUG - New I/O worker #970] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.339 [DEBUG - pool-3-thread-51] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.339 [DEBUG - New I/O worker #1200] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.339 [DEBUG - New I/O worker #970] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.339 [DEBUG - New I/O worker #1042] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.340 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.341 [DEBUG - New I/O worker #1620] (Negotiator.java:764) Authenticated connection [id: 0x73c9c54d, /127.1.193.124:46526 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.343 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.343 [DEBUG - New I/O worker #1601] (Negotiator.java:764) Authenticated connection [id: 0x5e834693, /127.1.193.126:51337 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.347 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.347 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.348 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.348 [DEBUG - New I/O worker #1672] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.348 [DEBUG - New I/O worker #1672] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.349 [DEBUG - pool-3-thread-47] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.349 [DEBUG - New I/O worker #1679] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.349 [DEBUG - New I/O worker #1677] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.350 [DEBUG - New I/O worker #1672] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.351 [DEBUG - New I/O worker #1597] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.352 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.352 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.353 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.353 [DEBUG - New I/O worker #1617] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.353 [DEBUG - New I/O worker #1617] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.355 [DEBUG - pool-3-thread-68] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.355 [DEBUG - New I/O worker #1620] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.355 [DEBUG - New I/O worker #1623] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.356 [DEBUG - New I/O worker #1617] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.360 [DEBUG - New I/O worker #1597] (Negotiator.java:764) Authenticated connection [id: 0xc2bcdfd5, /127.1.193.125:50598 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.363 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.364 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=267), Traces: [0ms] refreshing cache from master, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [239ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [251ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [267ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.364 [DEBUG - New I/O worker #1597] (Deferred.java:1330) callback=retry RPC@371172701 returned Deferred@304096070(state=PENDING, result=null, callback=(continuation of Deferred@1461890918 after retry RPC@371172701), errback=(continuation of Deferred@1461890918 after retry RPC@371172701)), so the following Deferred is getting paused: Deferred@1461890918(state=PAUSED, result=Deferred@304096070, callback=org.apache.kudu.client.AsyncKuduClient$6@de1c44c -> wakeup thread pool-3-thread-7, errback=passthrough -> wakeup thread pool-3-thread-7) 01:26:52.366 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.370 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.370 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.371 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.371 [DEBUG - New I/O worker #1597] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.372 [DEBUG - New I/O worker #1597] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.372 [DEBUG - pool-3-thread-7] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.373 [DEBUG - New I/O worker #1599] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.373 [DEBUG - New I/O worker #1601] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.373 [DEBUG - New I/O worker #1597] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.398 [DEBUG - pool-3-thread-18] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.412 [DEBUG - New I/O worker #1634] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.413 [DEBUG - New I/O worker #1639] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.415 [DEBUG - New I/O worker #1636] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.415 [DEBUG - pool-3-thread-49] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.420 [DEBUG - New I/O worker #1634] (Negotiator.java:764) Authenticated connection [id: 0x5c9c5230, /127.1.193.125:50604 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.422 [DEBUG - New I/O worker #1636] (Negotiator.java:764) Authenticated connection [id: 0x0e0051af, /127.1.193.124:46529 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.424 [DEBUG - New I/O worker #1639] (Negotiator.java:764) Authenticated connection [id: 0xa025c429, /127.1.193.126:51343 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.425 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.425 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=121), Traces: [0ms] refreshing cache from master, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [119ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:52.426 [DEBUG - New I/O worker #1634] (Deferred.java:1330) callback=retry RPC@210314053 returned Deferred@146187788(state=PENDING, result=null, callback=(continuation of Deferred@1303652734 after retry RPC@210314053), errback=(continuation of Deferred@1303652734 after retry RPC@210314053)), so the following Deferred is getting paused: Deferred@1303652734(state=PAUSED, result=Deferred@146187788, callback=org.apache.kudu.client.AsyncKuduClient$6@39652899 -> wakeup thread pool-3-thread-21, errback=passthrough -> wakeup thread pool-3-thread-21) 01:26:52.428 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.432 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.432 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.432 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.432 [DEBUG - New I/O worker #1634] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.433 [DEBUG - New I/O worker #1634] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.433 [DEBUG - pool-3-thread-57] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.434 [DEBUG - pool-3-thread-21] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.434 [DEBUG - New I/O worker #1636] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.435 [DEBUG - New I/O worker #1639] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.435 [DEBUG - New I/O worker #1634] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.438 [DEBUG - pool-3-thread-5] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.456 [DEBUG - pool-3-thread-78] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.458 [DEBUG - pool-3-thread-17] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.459 [DEBUG - pool-3-thread-50] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.461 [DEBUG - New I/O worker #1586] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.462 [DEBUG - New I/O worker #1590] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.464 [DEBUG - New I/O worker #1588] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.470 [DEBUG - New I/O worker #1586] (Negotiator.java:764) Authenticated connection [id: 0x5fb58165, /127.1.193.125:50607 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.474 [DEBUG - New I/O worker #1588] (Negotiator.java:764) Authenticated connection [id: 0x4d22faa4, /127.1.193.124:46532 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.474 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.476 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=17), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [15ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.474 [DEBUG - New I/O worker #1590] (Negotiator.java:764) Authenticated connection [id: 0x9e03985c, /127.1.193.126:51346 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.483 [DEBUG - New I/O worker #1691] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.485 [DEBUG - New I/O worker #1747] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.485 [DEBUG - New I/O worker #1876] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.479 [DEBUG - New I/O worker #1586] (Deferred.java:1330) callback=retry RPC@367675387 returned Deferred@299225502(state=PENDING, result=null, callback=(continuation of Deferred@1423396288 after retry RPC@367675387), errback=(continuation of Deferred@1423396288 after retry RPC@367675387)), so the following Deferred is getting paused: Deferred@1423396288(state=PAUSED, result=Deferred@299225502, callback=org.apache.kudu.client.AsyncKuduClient$6@225b677e -> wakeup thread pool-3-thread-17, errback=passthrough -> wakeup thread pool-3-thread-17) 01:26:52.477 [DEBUG - New I/O worker #1738] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.494 [DEBUG - New I/O worker #1876] (Negotiator.java:764) Authenticated connection [id: 0x555663df, /127.1.193.126:51352 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.477 [DEBUG - New I/O worker #1622] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.476 [DEBUG - New I/O worker #1754] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.500 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.504 [DEBUG - New I/O worker #1754] (Negotiator.java:764) Authenticated connection [id: 0xc0f331f0, /127.1.193.126:51349 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.503 [DEBUG - New I/O worker #1738] (Negotiator.java:764) Authenticated connection [id: 0xa9f7d39d, /127.1.193.124:46535 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.507 [DEBUG - New I/O worker #1622] (Negotiator.java:764) Authenticated connection [id: 0xcdaa0564, /127.1.193.125:50610 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.510 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.510 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.511 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=52), Traces: [0ms] refreshing cache from master, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [50ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [51ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [51ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.511 [DEBUG - New I/O worker #1622] (Deferred.java:1330) callback=retry RPC@1143760161 returned Deferred@1075050876(state=PENDING, result=null, callback=(continuation of Deferred@85028634 after retry RPC@1143760161), errback=(continuation of Deferred@85028634 after retry RPC@1143760161)), so the following Deferred is getting paused: Deferred@85028634(state=PAUSED, result=Deferred@1075050876, callback=org.apache.kudu.client.AsyncKuduClient$6@137cf646 -> wakeup thread pool-3-thread-50, errback=passthrough -> wakeup thread pool-3-thread-50) 01:26:52.514 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.514 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.515 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.519 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.520 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.520 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.520 [DEBUG - New I/O worker #1622] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.520 [DEBUG - New I/O worker #1586] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.521 [DEBUG - New I/O worker #1622] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.522 [DEBUG - pool-3-thread-50] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.522 [DEBUG - New I/O worker #1754] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.523 [DEBUG - New I/O worker #1622] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.524 [DEBUG - New I/O worker #1738] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.524 [DEBUG - New I/O worker #1586] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.525 [DEBUG - pool-3-thread-17] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.525 [DEBUG - New I/O worker #1588] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.526 [DEBUG - New I/O worker #1590] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.526 [DEBUG - New I/O worker #1586] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.529 [DEBUG - pool-3-thread-45] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.568 [DEBUG - pool-3-thread-14] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.569 [DEBUG - New I/O worker #1663] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.569 [DEBUG - New I/O worker #1819] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.569 [DEBUG - New I/O worker #1718] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.576 [DEBUG - New I/O worker #1663] (Negotiator.java:764) Authenticated connection [id: 0x1cdcb2a9, /127.1.193.125:50616 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.577 [DEBUG - New I/O worker #1819] (Negotiator.java:764) Authenticated connection [id: 0x4294f681, /127.1.193.126:51355 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.580 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.580 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=142), Traces: [0ms] refreshing cache from master, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [142ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:52.581 [DEBUG - New I/O worker #1663] (Deferred.java:1330) callback=retry RPC@195353247 returned Deferred@261836595(state=PENDING, result=null, callback=(continuation of Deferred@1251594404 after retry RPC@195353247), errback=(continuation of Deferred@1251594404 after retry RPC@195353247)), so the following Deferred is getting paused: Deferred@1251594404(state=PAUSED, result=Deferred@261836595, callback=org.apache.kudu.client.AsyncKuduClient$6@56c4b488 -> wakeup thread pool-3-thread-5, errback=passthrough -> wakeup thread pool-3-thread-5) 01:26:52.583 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.586 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.587 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.588 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.589 [DEBUG - New I/O worker #1810] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.589 [DEBUG - New I/O worker #1669] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.589 [DEBUG - New I/O worker #1717] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.590 [DEBUG - New I/O worker #1663] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.591 [DEBUG - New I/O worker #1663] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.592 [DEBUG - pool-3-thread-5] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.593 [DEBUG - New I/O worker #1663] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.593 [DEBUG - New I/O worker #1819] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:52.593688 4457 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.567642 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46541 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568182 (+ 540us) server_negotiation.cc:184] Beginning negotiation 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568207 (+ 25us) server_negotiation.cc:373] Waiting for connection header 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568252 (+ 45us) server_negotiation.cc:381] Connection header received 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568381 (+ 129us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568409 (+ 28us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568555 (+ 146us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.568738 (+ 183us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.573219 (+ 4481us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.573795 (+ 576us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.592627 (+ 18832us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46541: BlockingRecv error: recv got EOF from 127.1.193.124:46541 (error 108) 01:26:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":312} 01:26:52.596 [DEBUG - New I/O worker #1810] (Negotiator.java:764) Authenticated connection [id: 0x11c6ea6e, /127.1.193.126:51358 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.602 [DEBUG - New I/O worker #1669] (Negotiator.java:764) Authenticated connection [id: 0xea8f1fb6, /127.1.193.125:50619 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.602 [DEBUG - pool-3-thread-35] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.602 [DEBUG - New I/O worker #1717] (Negotiator.java:764) Authenticated connection [id: 0x419a12e2, /127.1.193.124:46544 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.606 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.606 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=172), Traces: [0ms] refreshing cache from master, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [166ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.606 [DEBUG - New I/O worker #1669] (Deferred.java:1330) callback=retry RPC@606157439 returned Deferred@538884643(state=PENDING, result=null, callback=(continuation of Deferred@1696346180 after retry RPC@606157439), errback=(continuation of Deferred@1696346180 after retry RPC@606157439)), so the following Deferred is getting paused: Deferred@1696346180(state=PAUSED, result=Deferred@538884643, callback=org.apache.kudu.client.AsyncKuduClient$6@79ec19fd -> wakeup thread pool-3-thread-57, errback=passthrough -> wakeup thread pool-3-thread-57) 01:26:52.610 [DEBUG - New I/O worker #1766] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.611 [DEBUG - New I/O worker #1776] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.612 [DEBUG - New I/O worker #1687] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.613 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.616 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.617 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.617 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.617 [DEBUG - New I/O worker #1669] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.618 [DEBUG - New I/O worker #1669] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.620 [DEBUG - New I/O worker #1669] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.620 [DEBUG - New I/O worker #1810] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.620 [DEBUG - pool-3-thread-57] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.621 [DEBUG - New I/O worker #1717] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.623 [DEBUG - New I/O worker #1766] (Negotiator.java:764) Authenticated connection [id: 0x34ca0881, /127.1.193.124:46547 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.625 [DEBUG - New I/O worker #1687] (Negotiator.java:764) Authenticated connection [id: 0x822e08ca, /127.1.193.125:50622 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.625 [DEBUG - New I/O worker #1776] (Negotiator.java:764) Authenticated connection [id: 0xa047eae2, /127.1.193.126:51361 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.629 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.630 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=214), Traces: [0ms] refreshing cache from master, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [211ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [213ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [214ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.630 [DEBUG - New I/O worker #1687] (Deferred.java:1330) callback=retry RPC@1021932458 returned Deferred@953610479(state=PENDING, result=null, callback=(continuation of Deferred@2111076753 after retry RPC@1021932458), errback=(continuation of Deferred@2111076753 after retry RPC@1021932458)), so the following Deferred is getting paused: Deferred@2111076753(state=PAUSED, result=Deferred@953610479, callback=org.apache.kudu.client.AsyncKuduClient$6@3be0e7d7 -> wakeup thread pool-3-thread-49, errback=passthrough -> wakeup thread pool-3-thread-49) 01:26:52.634 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.638 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.638 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.638 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.639 [DEBUG - New I/O worker #1687] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.640 [DEBUG - New I/O worker #1687] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.644 [DEBUG - pool-3-thread-49] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.645 [DEBUG - New I/O worker #1766] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.645 [DEBUG - New I/O worker #1687] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.645 [DEBUG - New I/O worker #1776] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.666 [DEBUG - pool-3-thread-63] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.677 [DEBUG - New I/O worker #1699] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.678 [DEBUG - New I/O worker #1698] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.678 [DEBUG - New I/O worker #1696] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.689 [DEBUG - New I/O worker #1696] (Negotiator.java:764) Authenticated connection [id: 0x4e6b14a7, /127.1.193.125:50625 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.691 [DEBUG - New I/O worker #1762] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.690 [DEBUG - New I/O worker #1708] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.691 [DEBUG - New I/O worker #1705] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.693 [DEBUG - New I/O worker #1699] (Negotiator.java:764) Authenticated connection [id: 0x7fba3a85, /127.1.193.126:51364 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.695 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.696 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=297), Traces: [0ms] refreshing cache from master, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [296ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.696 [DEBUG - New I/O worker #1696] (Deferred.java:1330) callback=retry RPC@1606992389 returned Deferred@1542937892(state=PENDING, result=null, callback=(continuation of Deferred@519424062 after retry RPC@1606992389), errback=(continuation of Deferred@519424062 after retry RPC@1606992389)), so the following Deferred is getting paused: Deferred@519424062(state=PAUSED, result=Deferred@1542937892, callback=org.apache.kudu.client.AsyncKuduClient$6@3660f022 -> wakeup thread pool-3-thread-18, errback=passthrough -> wakeup thread pool-3-thread-18) 01:26:52.699 [DEBUG - New I/O worker #1705] (Negotiator.java:764) Authenticated connection [id: 0x812cdf68, /127.1.193.125:50628 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.699 [DEBUG - New I/O worker #1708] (Negotiator.java:764) Authenticated connection [id: 0x21852269, /127.1.193.124:46553 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.700 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.703 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.703 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=36), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [35ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.705 [DEBUG - New I/O worker #1762] (Negotiator.java:764) Authenticated connection [id: 0xf92210b6, /127.1.193.126:51367 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.706 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.706 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.706 [DEBUG - New I/O worker #1705] (Deferred.java:1330) callback=retry RPC@12219294 returned Deferred@75888465(state=PENDING, result=null, callback=(continuation of Deferred@1099397541 after retry RPC@12219294), errback=(continuation of Deferred@1099397541 after retry RPC@12219294)), so the following Deferred is getting paused: Deferred@1099397541(state=PAUSED, result=Deferred@75888465, callback=org.apache.kudu.client.AsyncKuduClient$6@23797920 -> wakeup thread pool-3-thread-63, errback=passthrough -> wakeup thread pool-3-thread-63) 01:26:52.707 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.708 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.708 [DEBUG - New I/O worker #1696] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.709 [DEBUG - New I/O worker #1696] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.710 [DEBUG - pool-3-thread-18] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.710 [DEBUG - New I/O worker #1699] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.711 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.712 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.712 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.712 [DEBUG - New I/O worker #1705] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.713 [DEBUG - New I/O worker #1705] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:52.713011 4567 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.675737 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46550 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.676685 (+ 948us) server_negotiation.cc:184] Beginning negotiation 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.676713 (+ 28us) server_negotiation.cc:373] Waiting for connection header 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.676762 (+ 49us) server_negotiation.cc:381] Connection header received 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.676929 (+ 167us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.676959 (+ 30us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.677101 (+ 142us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.677266 (+ 165us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.679776 (+ 2510us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.680352 (+ 576us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.684245 (+ 3893us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.685399 (+ 1154us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.685618 (+ 219us) server_negotiation.cc:584] Negotiated auth-only TLSv1 with cipher AES256-SHA SSLv3 Kx=RSA Au=RSA Enc=AES(256) Mac=SHA1 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:52.710971 (+ 25353us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46550: BlockingRecv error: recv got EOF from 127.1.193.124:46550 (error 108) 01:26:52.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":673} 01:26:52.714 [DEBUG - New I/O worker #1696] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.714 [DEBUG - New I/O worker #1705] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.715 [DEBUG - New I/O worker #1762] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.714 [DEBUG - New I/O worker #1708] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.714 [DEBUG - pool-3-thread-63] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.720 [DEBUG - pool-3-thread-1] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.725 [DEBUG - pool-3-thread-62] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.730 [DEBUG - New I/O worker #1674] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.731 [DEBUG - New I/O worker #1797] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.731 [DEBUG - New I/O worker #1715] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.731 [DEBUG - pool-3-thread-64] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.738 [DEBUG - New I/O worker #1674] (Negotiator.java:764) Authenticated connection [id: 0xa467e9c8, /127.1.193.125:50631 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.739 [DEBUG - New I/O worker #1797] (Negotiator.java:764) Authenticated connection [id: 0x6ab0cd56, /127.1.193.126:51370 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.740 [DEBUG - pool-3-thread-70] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.741 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.742 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=138), Traces: [0ms] refreshing cache from master, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [138ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [139ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:52.742 [DEBUG - New I/O worker #1715] (Negotiator.java:764) Authenticated connection [id: 0x44bb24e5, /127.1.193.124:46556 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.745 [DEBUG - New I/O worker #1674] (Deferred.java:1330) callback=retry RPC@2105039894 returned Deferred@2034751329(state=PENDING, result=null, callback=(continuation of Deferred@1011177005 after retry RPC@2105039894), errback=(continuation of Deferred@1011177005 after retry RPC@2105039894)), so the following Deferred is getting paused: Deferred@1011177005(state=PAUSED, result=Deferred@2034751329, callback=org.apache.kudu.client.AsyncKuduClient$6@32143a80 -> wakeup thread pool-3-thread-35, errback=passthrough -> wakeup thread pool-3-thread-35) 01:26:52.746 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.750 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.750 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.751 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.751 [DEBUG - New I/O worker #1674] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.752 [DEBUG - New I/O worker #1674] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.752 [DEBUG - pool-3-thread-35] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.753 [DEBUG - New I/O worker #1715] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.753 [DEBUG - New I/O worker #1797] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.753 [DEBUG - New I/O worker #1674] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.758 [DEBUG - New I/O worker #1721] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.758 [DEBUG - New I/O worker #1653] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.758 [DEBUG - New I/O worker #1835] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.768 [DEBUG - New I/O worker #1835] (Negotiator.java:764) Authenticated connection [id: 0x71663d1e, /127.1.193.126:51373 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.768 [DEBUG - New I/O worker #1653] (Negotiator.java:764) Authenticated connection [id: 0xf98d1e3f, /127.1.193.125:50634 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.772 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.772 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=203), Traces: [0ms] refreshing cache from master, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [187ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [202ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.773 [DEBUG - New I/O worker #1653] (Deferred.java:1330) callback=retry RPC@1453794411 returned Deferred@1385740903(state=PENDING, result=null, callback=(continuation of Deferred@395980368 after retry RPC@1453794411), errback=(continuation of Deferred@395980368 after retry RPC@1453794411)), so the following Deferred is getting paused: Deferred@395980368(state=PAUSED, result=Deferred@1385740903, callback=org.apache.kudu.client.AsyncKuduClient$6@492cbddd -> wakeup thread pool-3-thread-14, errback=passthrough -> wakeup thread pool-3-thread-14) 01:26:52.775 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.775 [DEBUG - New I/O worker #1721] (Negotiator.java:764) Authenticated connection [id: 0x43e47da3, /127.1.193.124:46559 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.779 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.780 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.780 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.780 [DEBUG - New I/O worker #1653] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.781 [DEBUG - New I/O worker #1653] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.782 [DEBUG - pool-3-thread-14] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.782 [DEBUG - New I/O worker #1653] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:52.783157 1949 connection.cc:554] Connection torn down before Call kudu.master.MasterService.ConnectToMaster from 127.1.193.124:46559 (request call id 0) could send its response 01:26:52.783 [DEBUG - New I/O worker #1835] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.784 [DEBUG - New I/O worker #1721] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.784 [INFO - New I/O worker #1721] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:52.804 [DEBUG - pool-3-thread-58] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.806 [DEBUG - pool-3-thread-26] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.813 [DEBUG - pool-3-thread-16] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.813 [DEBUG - pool-3-thread-23] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.822 [DEBUG - New I/O worker #1750] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.824 [DEBUG - New I/O worker #1680] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.824 [DEBUG - New I/O worker #1748] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.831 [DEBUG - pool-3-thread-69] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.831 [DEBUG - New I/O worker #1680] (Negotiator.java:764) Authenticated connection [id: 0xac8d113d, /127.1.193.125:50637 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.832 [DEBUG - New I/O worker #1750] (Negotiator.java:764) Authenticated connection [id: 0x0a3bd20a, /127.1.193.126:51376 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.833 [DEBUG - New I/O worker #1748] (Negotiator.java:764) Authenticated connection [id: 0xd444519b, /127.1.193.124:46562 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.836 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.837 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=307), Traces: [0ms] refreshing cache from master, [290ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [290ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [290ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [307ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [307ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:52.840 [DEBUG - New I/O worker #1680] (Deferred.java:1330) callback=retry RPC@1225661471 returned Deferred@1295094246(state=PENDING, result=null, callback=(continuation of Deferred@137565732 after retry RPC@1225661471), errback=(continuation of Deferred@137565732 after retry RPC@1225661471)), so the following Deferred is getting paused: Deferred@137565732(state=PAUSED, result=Deferred@1295094246, callback=org.apache.kudu.client.AsyncKuduClient$6@a640fe -> wakeup thread pool-3-thread-45, errback=passthrough -> wakeup thread pool-3-thread-45) 01:26:52.841 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.846 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.846 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.847 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.847 [DEBUG - New I/O worker #1680] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.847 [DEBUG - New I/O worker #1680] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.848 [DEBUG - pool-3-thread-45] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.848 [DEBUG - New I/O worker #1748] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.848 [DEBUG - New I/O worker #1750] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.856 [DEBUG - New I/O worker #1747] (Negotiator.java:764) Authenticated connection [id: 0x4ed37c7b, /127.1.193.124:46538 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.856 [INFO - New I/O worker #1698] (Connection.java:435) [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.858 [DEBUG - New I/O worker #1698] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.856 [INFO - New I/O worker #1718] (Connection.java:435) [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.858 [DEBUG - New I/O worker #1718] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.859 [INFO - New I/O worker #1718] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.854 [DEBUG - pool-3-thread-3] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.848 [DEBUG - New I/O worker #1680] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.854 [DEBUG - pool-3-thread-36] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.854 [DEBUG - pool-3-thread-2] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.852 [DEBUG - New I/O worker #1691] (Negotiator.java:764) Authenticated connection [id: 0xb8aa8658, /127.1.193.125:50613 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.858 [INFO - New I/O worker #1698] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: [peer master-127.1.193.124:58247(127.1.193.124:58247)] disconnected from peer 01:26:52.864 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.864 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=407), Traces: [0ms] refreshing cache from master, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [41ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [404ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [407ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.865 [DEBUG - New I/O worker #1691] (Deferred.java:1330) callback=retry RPC@909292501 returned Deferred@839721419(state=PENDING, result=null, callback=(continuation of Deferred@1997519342 after retry RPC@909292501), errback=(continuation of Deferred@1997519342 after retry RPC@909292501)), so the following Deferred is getting paused: Deferred@1997519342(state=PAUSED, result=Deferred@839721419, callback=org.apache.kudu.client.AsyncKuduClient$6@653d9526 -> wakeup thread pool-3-thread-78, errback=passthrough -> wakeup thread pool-3-thread-78) 01:26:52.867 [DEBUG - pool-3-thread-29] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.867 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.869 [DEBUG - pool-3-thread-37] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.870 [DEBUG - New I/O worker #1665] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.871 [DEBUG - New I/O worker #1874] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.871 [DEBUG - New I/O worker #1744] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.873 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.874 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.874 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.874 [DEBUG - New I/O worker #1691] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.875 [DEBUG - New I/O worker #1691] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.875 [DEBUG - pool-3-thread-78] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.875 [DEBUG - New I/O worker #1876] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.876 [DEBUG - New I/O worker #1691] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.876 [DEBUG - New I/O worker #1747] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.880 [DEBUG - New I/O worker #1665] (Negotiator.java:764) Authenticated connection [id: 0xa35d9027, /127.1.193.125:50640 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.881 [DEBUG - New I/O worker #1874] (Negotiator.java:764) Authenticated connection [id: 0xa2dbaf3d, /127.1.193.126:51379 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.883 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.883 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=51), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [51ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.883 [DEBUG - New I/O worker #1665] (Deferred.java:1330) callback=retry RPC@1553454630 returned Deferred@1487427948(state=PENDING, result=null, callback=(continuation of Deferred@497736733 after retry RPC@1553454630), errback=(continuation of Deferred@497736733 after retry RPC@1553454630)), so the following Deferred is getting paused: Deferred@497736733(state=PAUSED, result=Deferred@1487427948, callback=org.apache.kudu.client.AsyncKuduClient$6@4e39cd66 -> wakeup thread pool-3-thread-69, errback=passthrough -> wakeup thread pool-3-thread-69) 01:26:52.885 [DEBUG - New I/O worker #1744] (Negotiator.java:764) Authenticated connection [id: 0x85d50484, /127.1.193.124:46565 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.885 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.888 [DEBUG - New I/O worker #1752] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.890 [DEBUG - New I/O worker #1743] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.891 [DEBUG - New I/O worker #1664] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.895 [DEBUG - New I/O worker #1739] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.896 [DEBUG - New I/O worker #1637] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.897 [DEBUG - New I/O worker #1866] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.902 [DEBUG - New I/O worker #1739] (Negotiator.java:764) Authenticated connection [id: 0x41d50eb6, /127.1.193.124:46571 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.903 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.903 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.904 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.904 [DEBUG - New I/O worker #1665] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.904 [DEBUG - New I/O worker #1665] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.904 [DEBUG - New I/O worker #1637] (Negotiator.java:764) Authenticated connection [id: 0xfe16cbba, /127.1.193.125:50646 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.905 [DEBUG - pool-3-thread-69] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.905 [DEBUG - New I/O worker #1665] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.906 [DEBUG - New I/O worker #1874] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.906 [DEBUG - New I/O worker #1664] (Negotiator.java:764) Authenticated connection [id: 0xaf5e10c9, /127.1.193.125:50643 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:52.906489 1951 connection.cc:647] server connection from 127.1.193.124:46565 recv error: Network error: recv error from 0.0.0.0:0: Transport endpoint is not connected (error 107) 01:26:52.907 [DEBUG - New I/O worker #1752] (Negotiator.java:764) Authenticated connection [id: 0x8e0e14d2, /127.1.193.126:51382 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.908 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.910 [DEBUG - New I/O worker #1743] (Negotiator.java:764) Authenticated connection [id: 0x17a8162f, /127.1.193.124:46568 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.909 [DEBUG - New I/O worker #1744] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.912 [INFO - New I/O worker #1744] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:52.909 [DEBUG - New I/O worker #1866] (Negotiator.java:764) Authenticated connection [id: 0x21dd2a23, /127.1.193.126:51385 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.912 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.913 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=99), Traces: [0ms] refreshing cache from master, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [96ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [96ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.912 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=98), Traces: [0ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [94ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [95ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.914 [DEBUG - pool-3-thread-66] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.915 [DEBUG - New I/O worker #1637] (Deferred.java:1330) callback=retry RPC@1809740083 returned Deferred@1877080208(state=PENDING, result=null, callback=(continuation of Deferred@719551240 after retry RPC@1809740083), errback=(continuation of Deferred@719551240 after retry RPC@1809740083)), so the following Deferred is getting paused: Deferred@719551240(state=PAUSED, result=Deferred@1877080208, callback=org.apache.kudu.client.AsyncKuduClient$6@34c6b27 -> wakeup thread pool-3-thread-16, errback=passthrough -> wakeup thread pool-3-thread-16) 01:26:52.915 [DEBUG - New I/O worker #1664] (Deferred.java:1330) callback=retry RPC@2082767459 returned Deferred@2015037759(state=PENDING, result=null, callback=(continuation of Deferred@1025083480 after retry RPC@2082767459), errback=(continuation of Deferred@1025083480 after retry RPC@2082767459)), so the following Deferred is getting paused: Deferred@1025083480(state=PAUSED, result=Deferred@2015037759, callback=org.apache.kudu.client.AsyncKuduClient$6@540164ae -> wakeup thread pool-3-thread-23, errback=passthrough -> wakeup thread pool-3-thread-23) 01:26:52.916 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.917 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.921 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.922 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.922 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.923 [DEBUG - New I/O worker #1637] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.923 [DEBUG - New I/O worker #1637] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.922 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.924 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.924 [DEBUG - New I/O worker #1739] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.924 [DEBUG - pool-3-thread-16] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.924 [DEBUG - New I/O worker #1637] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.924 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.924 [DEBUG - New I/O worker #1866] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.926 [DEBUG - New I/O worker #1664] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.926 [DEBUG - New I/O worker #1664] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.927 [DEBUG - pool-3-thread-23] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.927 [DEBUG - New I/O worker #1752] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.927 [DEBUG - New I/O worker #1743] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.927 [DEBUG - New I/O worker #1664] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.937 [DEBUG - New I/O worker #1606] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.938 [DEBUG - New I/O worker #1604] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.939 [DEBUG - New I/O worker #1735] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.945 [DEBUG - New I/O worker #1604] (Negotiator.java:764) Authenticated connection [id: 0x6f977afe, /127.1.193.125:50649 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.949 [DEBUG - New I/O worker #1735] (Negotiator.java:764) Authenticated connection [id: 0xefd616ad, /127.1.193.126:51388 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.950 [DEBUG - New I/O worker #1606] (Negotiator.java:764) Authenticated connection [id: 0x6e123a63, /127.1.193.124:46574 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.950 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.951 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=144), Traces: [0ms] refreshing cache from master, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [143ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.956 [DEBUG - New I/O worker #1604] (Deferred.java:1330) callback=retry RPC@1000833722 returned Deferred@1066926358(state=PENDING, result=null, callback=(continuation of Deferred@2056947841 after retry RPC@1000833722), errback=(continuation of Deferred@2056947841 after retry RPC@1000833722)), so the following Deferred is getting paused: Deferred@2056947841(state=PAUSED, result=Deferred@1066926358, callback=org.apache.kudu.client.AsyncKuduClient$6@4c1feeee -> wakeup thread pool-3-thread-26, errback=passthrough -> wakeup thread pool-3-thread-26) 01:26:52.958 [DEBUG - New I/O worker #1851] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.958 [DEBUG - New I/O worker #1850] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:52.957 [DEBUG - New I/O worker #1612] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:52.965 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.967 [DEBUG - New I/O worker #1612] (Negotiator.java:764) Authenticated connection [id: 0x22c4e44a, /127.1.193.125:50652 => /127.1.193.125:37435] using SASL/PLAIN 01:26:52.966 [DEBUG - New I/O worker #1850] (Negotiator.java:764) Authenticated connection [id: 0x36e6fb8e, /127.1.193.124:46577 => /127.1.193.124:58247] using SASL/PLAIN 01:26:52.971 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:52.971 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.972 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=166), Traces: [0ms] refreshing cache from master, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [165ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [166ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:52.972 [DEBUG - New I/O worker #1612] (Deferred.java:1330) callback=retry RPC@298503752 returned Deferred@368400553(state=PENDING, result=null, callback=(continuation of Deferred@1358414963 after retry RPC@298503752), errback=(continuation of Deferred@1358414963 after retry RPC@298503752)), so the following Deferred is getting paused: Deferred@1358414963(state=PAUSED, result=Deferred@368400553, callback=org.apache.kudu.client.AsyncKuduClient$6@2612b64f -> wakeup thread pool-3-thread-58, errback=passthrough -> wakeup thread pool-3-thread-58) 01:26:52.973 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.973 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.976 [DEBUG - New I/O worker #1604] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.976 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:52.977 [DEBUG - New I/O worker #1851] (Negotiator.java:764) Authenticated connection [id: 0xdba20d18, /127.1.193.126:51391 => /127.1.193.126:40739] using SASL/PLAIN 01:26:52.976 [DEBUG - New I/O worker #1604] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.982 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:52.982 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:52.982 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:52.983 [DEBUG - New I/O worker #1612] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:52.983 [DEBUG - New I/O worker #1612] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:52.984 [DEBUG - pool-3-thread-58] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.984 [DEBUG - pool-3-thread-26] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:52.984 [DEBUG - New I/O worker #1735] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.984 [DEBUG - New I/O worker #1850] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.985 [DEBUG - New I/O worker #1851] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:52.985 [DEBUG - New I/O worker #1612] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.984 [DEBUG - New I/O worker #1606] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:52.986 [DEBUG - New I/O worker #1604] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:52.991 [DEBUG - pool-3-thread-73] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:52.999 [DEBUG - New I/O worker #1666] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:52.999 [DEBUG - New I/O worker #1803] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.000 [DEBUG - New I/O worker #1675] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.001 [DEBUG - New I/O worker #1662] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.001 [DEBUG - New I/O worker #1765] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.003 [DEBUG - New I/O worker #1660] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.012 [DEBUG - New I/O worker #1662] (Negotiator.java:764) Authenticated connection [id: 0xfff31628, /127.1.193.124:46582 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.013 [DEBUG - New I/O worker #1660] (Negotiator.java:764) Authenticated connection [id: 0xcd1cde5e, /127.1.193.125:50655 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.014 [DEBUG - New I/O worker #1765] (Negotiator.java:764) Authenticated connection [id: 0xfa20156b, /127.1.193.124:46581 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.015 [DEBUG - New I/O worker #1803] (Negotiator.java:764) Authenticated connection [id: 0x78b3648b, /127.1.193.126:51397 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.017 [DEBUG - New I/O worker #1675] (Negotiator.java:764) Authenticated connection [id: 0x92bb7995, /127.1.193.125:50656 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.018 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.019 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=286), Traces: [0ms] refreshing cache from master, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [286ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:53.020 [DEBUG - New I/O worker #1660] (Deferred.java:1330) callback=retry RPC@2140090836 returned Deferred@2075175677(state=PENDING, result=null, callback=(continuation of Deferred@1051868143 after retry RPC@2140090836), errback=(continuation of Deferred@1051868143 after retry RPC@2140090836)), so the following Deferred is getting paused: Deferred@1051868143(state=PAUSED, result=Deferred@2075175677, callback=org.apache.kudu.client.AsyncKuduClient$6@9c1c11d -> wakeup thread pool-3-thread-64, errback=passthrough -> wakeup thread pool-3-thread-64) 01:26:53.019 [DEBUG - New I/O worker #1666] (Negotiator.java:764) Authenticated connection [id: 0x050ec9d3, /127.1.193.126:51396 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.021 [DEBUG - pool-3-thread-72] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.022 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.022 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.022 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=281), Traces: [0ms] refreshing cache from master, [256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [278ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [280ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [281ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.025 [DEBUG - New I/O worker #1675] (Deferred.java:1330) callback=retry RPC@1512251725 returned Deferred@1578942161(state=PENDING, result=null, callback=(continuation of Deferred@454962038 after retry RPC@1512251725), errback=(continuation of Deferred@454962038 after retry RPC@1512251725)), so the following Deferred is getting paused: Deferred@454962038(state=PAUSED, result=Deferred@1578942161, callback=org.apache.kudu.client.AsyncKuduClient$6@6925ac69 -> wakeup thread pool-3-thread-70, errback=passthrough -> wakeup thread pool-3-thread-70) 01:26:53.026 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.031 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.032 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.032 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.032 [DEBUG - New I/O worker #1660] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.033 [DEBUG - New I/O worker #1660] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.034 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.034 [DEBUG - New I/O worker #1662] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.034 [DEBUG - pool-3-thread-64] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.034 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.035 [DEBUG - New I/O worker #1660] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.035 [DEBUG - New I/O worker #1666] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.035 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.036 [DEBUG - New I/O worker #1675] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.036 [DEBUG - New I/O worker #1675] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.037 [DEBUG - New I/O worker #1765] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.037 [DEBUG - New I/O worker #1675] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.037 [DEBUG - pool-3-thread-70] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.037 [DEBUG - New I/O worker #1803] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.047 [DEBUG - New I/O worker #1692] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.047 [DEBUG - New I/O worker #1792] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.050 [DEBUG - New I/O worker #1985] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.057 [DEBUG - New I/O worker #1792] (Negotiator.java:764) Authenticated connection [id: 0xf9290357, /127.1.193.124:46586 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.059 [DEBUG - New I/O worker #1692] (Negotiator.java:764) Authenticated connection [id: 0x286d6d7b, /127.1.193.125:50661 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.062 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.062 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=337), Traces: [0ms] refreshing cache from master, [318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [319ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [319ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [336ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.063 [DEBUG - New I/O worker #1692] (Deferred.java:1330) callback=retry RPC@990206892 returned Deferred@1060816916(state=PENDING, result=null, callback=(continuation of Deferred@2050511255 after retry RPC@990206892), errback=(continuation of Deferred@2050511255 after retry RPC@990206892)), so the following Deferred is getting paused: Deferred@2050511255(state=PAUSED, result=Deferred@1060816916, callback=org.apache.kudu.client.AsyncKuduClient$6@5ac11437 -> wakeup thread pool-3-thread-62, errback=passthrough -> wakeup thread pool-3-thread-62) 01:26:53.065 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.066 [DEBUG - New I/O worker #1985] (Negotiator.java:764) Authenticated connection [id: 0x7ac133e6, /127.1.193.126:51400 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.068 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.068 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.069 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.069 [DEBUG - New I/O worker #1692] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.069 [DEBUG - New I/O worker #1692] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.070 [DEBUG - New I/O worker #1692] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.070 [DEBUG - New I/O worker #1985] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.070 [DEBUG - pool-3-thread-62] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.070 [DEBUG - New I/O worker #1792] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.075 [DEBUG - New I/O worker #1683] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.076 [DEBUG - New I/O worker #1714] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.081 [DEBUG - pool-3-thread-76] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.084 [DEBUG - New I/O worker #1782] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.085 [DEBUG - New I/O worker #1714] (Negotiator.java:764) Authenticated connection [id: 0xe07ccbd7, /127.1.193.124:46589 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.085 [DEBUG - New I/O worker #1683] (Negotiator.java:764) Authenticated connection [id: 0xa311af5e, /127.1.193.125:50664 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.089 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.090 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=369), Traces: [0ms] refreshing cache from master, [353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [368ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [369ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.091 [DEBUG - New I/O worker #1683] (Deferred.java:1330) callback=retry RPC@1400195579 returned Deferred@1464522573(state=PENDING, result=null, callback=(continuation of Deferred@306725824 after retry RPC@1400195579), errback=(continuation of Deferred@306725824 after retry RPC@1400195579)), so the following Deferred is getting paused: Deferred@306725824(state=PAUSED, result=Deferred@1464522573, callback=org.apache.kudu.client.AsyncKuduClient$6@5ad356e0 -> wakeup thread pool-3-thread-1, errback=passthrough -> wakeup thread pool-3-thread-1) 01:26:53.090 [DEBUG - New I/O worker #1782] (Negotiator.java:764) Authenticated connection [id: 0x74c209d6, /127.1.193.126:51403 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.094 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.097 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.097 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.097 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.098 [DEBUG - New I/O worker #1683] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.098 [DEBUG - New I/O worker #1683] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.099 [DEBUG - pool-3-thread-1] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.099 [DEBUG - New I/O worker #1782] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.099 [DEBUG - New I/O worker #1683] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.099 [DEBUG - New I/O worker #1714] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.104 [DEBUG - pool-3-thread-80] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.124 [DEBUG - New I/O worker #1790] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.124 [DEBUG - New I/O worker #1789] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.125 [DEBUG - New I/O worker #1678] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.133 [DEBUG - New I/O worker #1848] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.133 [DEBUG - New I/O worker #1728] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.134 [DEBUG - New I/O worker #1618] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.137 [DEBUG - New I/O worker #1789] (Negotiator.java:764) Authenticated connection [id: 0xb5abee55, /127.1.193.124:46592 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.141 [DEBUG - New I/O worker #1678] (Negotiator.java:764) Authenticated connection [id: 0x165d5a81, /127.1.193.125:50667 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.143 [DEBUG - New I/O worker #1651] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.143 [DEBUG - New I/O worker #1649] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.144 [DEBUG - New I/O worker #1618] (Negotiator.java:764) Authenticated connection [id: 0x00cb8f71, /127.1.193.125:50670 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.143 [DEBUG - New I/O worker #1728] (Negotiator.java:764) Authenticated connection [id: 0xeebec6f7, /127.1.193.124:46595 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.146 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.146 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=42), Traces: [0ms] refreshing cache from master, [16ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [17ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [17ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [42ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.147 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.147 [DEBUG - New I/O worker #1678] (Deferred.java:1330) callback=retry RPC@974367961 returned Deferred@1043083075(state=PENDING, result=null, callback=(continuation of Deferred@2066654946 after retry RPC@974367961), errback=(continuation of Deferred@2066654946 after retry RPC@974367961)), so the following Deferred is getting paused: Deferred@2066654946(state=PAUSED, result=Deferred@1043083075, callback=org.apache.kudu.client.AsyncKuduClient$6@6dcd8068 -> wakeup thread pool-3-thread-80, errback=passthrough -> wakeup thread pool-3-thread-80) 01:26:53.147 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=155), Traces: [0ms] refreshing cache from master, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [155ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.148 [DEBUG - New I/O worker #1790] (Negotiator.java:764) Authenticated connection [id: 0x8751352f, /127.1.193.126:51406 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.150 [DEBUG - New I/O worker #1848] (Negotiator.java:764) Authenticated connection [id: 0xfe041bbd, /127.1.193.126:51409 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.152 [DEBUG - New I/O worker #1656] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.148 [DEBUG - New I/O worker #1618] (Deferred.java:1330) callback=retry RPC@1683825697 returned Deferred@1617079410(state=PENDING, result=null, callback=(continuation of Deferred@627060250 after retry RPC@1683825697), errback=(continuation of Deferred@627060250 after retry RPC@1683825697)), so the following Deferred is getting paused: Deferred@627060250(state=PAUSED, result=Deferred@1617079410, callback=org.apache.kudu.client.AsyncKuduClient$6@19fe5f63 -> wakeup thread pool-3-thread-73, errback=passthrough -> wakeup thread pool-3-thread-73) 01:26:53.153 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.153 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.158 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.158 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.159 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.159 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.159 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.160 [DEBUG - New I/O worker #1618] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.160 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.160 [DEBUG - New I/O worker #1618] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.160 [DEBUG - New I/O worker #1678] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.161 [DEBUG - New I/O worker #1678] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.161 [DEBUG - New I/O worker #1728] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.162 [DEBUG - pool-3-thread-80] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.162 [DEBUG - New I/O worker #1618] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.161 [DEBUG - New I/O worker #1656] (Negotiator.java:764) Authenticated connection [id: 0x7f22efa2, /127.1.193.126:51412 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.161 [DEBUG - pool-3-thread-73] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.163 [DEBUG - New I/O worker #1790] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.163 [DEBUG - New I/O worker #1789] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.163 [DEBUG - New I/O worker #1678] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.162 [DEBUG - New I/O worker #1848] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.177 [DEBUG - pool-3-thread-65] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.180 [DEBUG - pool-3-thread-12] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.186 [DEBUG - pool-3-thread-43] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.189 [DEBUG - pool-3-thread-61] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.190 [DEBUG - New I/O worker #1861] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.191 [DEBUG - New I/O worker #1753] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.192 [DEBUG - New I/O worker #1621] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.195 [DEBUG - New I/O worker #1684] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.197 [DEBUG - New I/O worker #1906] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.198 [DEBUG - New I/O worker #1745] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.207 [DEBUG - New I/O worker #1906] (Negotiator.java:764) Authenticated connection [id: 0x87733ab3, /127.1.193.126:51418 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.208 [DEBUG - New I/O worker #1745] (Negotiator.java:764) Authenticated connection [id: 0x589fe9bb, /127.1.193.124:46604 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.209 [DEBUG - New I/O worker #1753] (Negotiator.java:764) Authenticated connection [id: 0x58786527, /127.1.193.124:46601 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.208 [DEBUG - New I/O worker #1684] (Negotiator.java:764) Authenticated connection [id: 0x7af13a39, /127.1.193.125:50679 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.209 [DEBUG - New I/O worker #1861] (Negotiator.java:764) Authenticated connection [id: 0x451b258d, /127.1.193.126:51415 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.211 [DEBUG - New I/O worker #1621] (Negotiator.java:764) Authenticated connection [id: 0xfc35021f, /127.1.193.125:50676 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.214 [DEBUG - New I/O worker #1635] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.217 [DEBUG - New I/O worker #1576] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.219 [DEBUG - New I/O worker #1578] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.220 [DEBUG - New I/O worker #1577] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.221 [DEBUG - New I/O worker #1865] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.222 [DEBUG - New I/O worker #1751] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.228 [DEBUG - New I/O worker #1865] (Negotiator.java:764) Authenticated connection [id: 0x2fca9f92, /127.1.193.126:51421 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.230 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.231 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=315), Traces: [0ms] refreshing cache from master, [273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [305ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [305ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [316ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.231 [DEBUG - New I/O worker #1621] (Deferred.java:1330) callback=retry RPC@1929844259 returned Deferred@2000198811(state=PENDING, result=null, callback=(continuation of Deferred@842669080 after retry RPC@1929844259), errback=(continuation of Deferred@842669080 after retry RPC@1929844259)), so the following Deferred is getting paused: Deferred@842669080(state=PAUSED, result=Deferred@2000198811, callback=org.apache.kudu.client.AsyncKuduClient$6@48bfca1b -> wakeup thread pool-3-thread-66, errback=passthrough -> wakeup thread pool-3-thread-66) 01:26:53.231 [DEBUG - New I/O worker #1576] (Negotiator.java:764) Authenticated connection [id: 0x90283731, /127.1.193.125:50685 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.233 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.233 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=366), Traces: [0ms] refreshing cache from master, [325ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [325ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [325ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [350ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [366ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.233 [DEBUG - New I/O worker #1578] (Negotiator.java:764) Authenticated connection [id: 0x901db9ba, /127.1.193.126:51424 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.234 [DEBUG - New I/O worker #1751] (Negotiator.java:764) Authenticated connection [id: 0x78d77bfc, /127.1.193.124:46607 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.234 [DEBUG - New I/O worker #1684] (Deferred.java:1330) callback=retry RPC@907526178 returned Deferred@841499192(state=PENDING, result=null, callback=(continuation of Deferred@1999292953 after retry RPC@907526178), errback=(continuation of Deferred@1999292953 after retry RPC@907526178)), so the following Deferred is getting paused: Deferred@1999292953(state=PAUSED, result=Deferred@841499192, callback=org.apache.kudu.client.AsyncKuduClient$6@4e8937dd -> wakeup thread pool-3-thread-29, errback=passthrough -> wakeup thread pool-3-thread-29) 01:26:53.235 [DEBUG - New I/O worker #1635] (Negotiator.java:764) Authenticated connection [id: 0x52c4c869, /127.1.193.125:50682 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.234 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.237 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=367), Traces: [0ms] refreshing cache from master, [344ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [344ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [344ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [364ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [366ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, deferred=null) after lookup 01:26:53.238 [DEBUG - New I/O worker #1576] (Deferred.java:1330) callback=retry RPC@2075358748 returned Deferred@2139942322(state=PENDING, result=null, callback=(continuation of Deferred@982416423 after retry RPC@2075358748), errback=(continuation of Deferred@982416423 after retry RPC@2075358748)), so the following Deferred is getting paused: Deferred@982416423(state=PAUSED, result=Deferred@2139942322, callback=org.apache.kudu.client.AsyncKuduClient$6@6b1dbf22 -> wakeup thread pool-3-thread-37, errback=passthrough -> wakeup thread pool-3-thread-37) 01:26:53.239 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.240 [DEBUG - New I/O worker #1577] (Negotiator.java:764) Authenticated connection [id: 0x717a7ffa, /127.1.193.124:46610 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.243 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.243 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.243 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.244 [DEBUG - New I/O worker #1576] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.244 [DEBUG - New I/O worker #1576] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.244 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.245 [DEBUG - pool-3-thread-37] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.246 [DEBUG - New I/O worker #1576] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.246 [DEBUG - New I/O worker #1577] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.246 [DEBUG - New I/O worker #1578] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.249 [DEBUG - New I/O worker #1895] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.252 [DEBUG - New I/O worker #1583] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.252 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.253 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=63), Traces: [0ms] refreshing cache from master, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [43ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [54ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.253 [DEBUG - New I/O worker #1584] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.254 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.254 [DEBUG - New I/O worker #1881] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.253 [DEBUG - New I/O worker #1635] (Deferred.java:1330) callback=retry RPC@1800664231 returned Deferred@1869380885(state=PENDING, result=null, callback=(continuation of Deferred@711917212 after retry RPC@1800664231), errback=(continuation of Deferred@711917212 after retry RPC@1800664231)), so the following Deferred is getting paused: Deferred@711917212(state=PAUSED, result=Deferred@1869380885, callback=org.apache.kudu.client.AsyncKuduClient$6@6f0ce28a -> wakeup thread pool-3-thread-61, errback=passthrough -> wakeup thread pool-3-thread-61) 01:26:53.255 [DEBUG - New I/O worker #1581] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.256 [DEBUG - New I/O worker #1895] (Negotiator.java:764) Authenticated connection [id: 0x100ee0d7, /127.1.193.126:51427 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.256 [DEBUG - New I/O worker #1580] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.259 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.259 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.259 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.260 [DEBUG - New I/O worker #1621] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.260 [DEBUG - New I/O worker #1621] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.261 [DEBUG - New I/O worker #1621] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.261 [DEBUG - New I/O worker #1753] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.261 [DEBUG - pool-3-thread-66] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.261 [DEBUG - New I/O worker #1861] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.265 [DEBUG - New I/O worker #1583] (Negotiator.java:764) Authenticated connection [id: 0xfdcbaff5, /127.1.193.126:51430 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.268 [DEBUG - New I/O worker #1581] (Negotiator.java:764) Authenticated connection [id: 0x1118052d, /127.1.193.124:46616 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.268 [DEBUG - New I/O worker #1881] (Negotiator.java:764) Authenticated connection [id: 0x756e076a, /127.1.193.124:46613 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.270 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.270 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.271 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.271 [DEBUG - New I/O worker #1684] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.271 [DEBUG - New I/O worker #1684] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.272 [DEBUG - pool-3-thread-29] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.273 [DEBUG - New I/O worker #1745] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.273 [DEBUG - New I/O worker #1906] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.273 [DEBUG - New I/O worker #1684] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.274 [DEBUG - New I/O worker #1686] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.274 [DEBUG - New I/O worker #1749] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.275 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.279 [DEBUG - New I/O worker #1584] (Negotiator.java:764) Authenticated connection [id: 0xf11972ad, /127.1.193.125:50688 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.280 [DEBUG - New I/O worker #1746] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.282 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.282 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=421), Traces: [0ms] refreshing cache from master, [386ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [387ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [387ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [399ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [414ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [422ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.283 [DEBUG - New I/O worker #1584] (Deferred.java:1330) callback=retry RPC@294063402 returned Deferred@364415938(state=PENDING, result=null, callback=(continuation of Deferred@1354368785 after retry RPC@294063402), errback=(continuation of Deferred@1354368785 after retry RPC@294063402)), so the following Deferred is getting paused: Deferred@1354368785(state=PAUSED, result=Deferred@364415938, callback=org.apache.kudu.client.AsyncKuduClient$6@486738c4 -> wakeup thread pool-3-thread-2, errback=passthrough -> wakeup thread pool-3-thread-2) 01:26:53.282 [DEBUG - New I/O worker #1749] (Negotiator.java:764) Authenticated connection [id: 0xee13a4a9, /127.1.193.126:51433 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.283 [DEBUG - New I/O worker #1686] (Negotiator.java:764) Authenticated connection [id: 0xcfd09f52, /127.1.193.125:50694 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.285 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.286 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.287 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=427), Traces: [0ms] refreshing cache from master, [410ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [410ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [411ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [426ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [426ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.287 [DEBUG - New I/O worker #1686] (Deferred.java:1330) callback=retry RPC@1581169439 returned Deferred@1510024192(state=PENDING, result=null, callback=(continuation of Deferred@520335652 after retry RPC@1581169439), errback=(continuation of Deferred@520335652 after retry RPC@1581169439)), so the following Deferred is getting paused: Deferred@520335652(state=PAUSED, result=Deferred@1510024192, callback=org.apache.kudu.client.AsyncKuduClient$6@51bef7b -> wakeup thread pool-3-thread-3, errback=passthrough -> wakeup thread pool-3-thread-3) 01:26:53.288 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.289 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.289 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.289 [DEBUG - New I/O worker #1635] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.289 [DEBUG - New I/O worker #1635] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.290 [DEBUG - pool-3-thread-61] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.290 [DEBUG - New I/O worker #1751] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.291 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.291 [DEBUG - New I/O worker #1635] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.291 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.291 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.292 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.292 [DEBUG - New I/O worker #1584] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.292 [DEBUG - New I/O worker #1584] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.293 [DEBUG - New I/O worker #1865] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.293 [DEBUG - pool-3-thread-2] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.294 [DEBUG - New I/O worker #1895] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.294 [DEBUG - New I/O worker #1584] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.294 [DEBUG - New I/O worker #1881] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.299 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.300 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.300 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.300 [DEBUG - New I/O worker #1686] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.301 [DEBUG - New I/O worker #1686] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.301 [DEBUG - pool-3-thread-3] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.301 [DEBUG - New I/O worker #1749] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.301 [DEBUG - New I/O worker #1746] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: connection disconnected 01:26:53.302 [INFO - New I/O worker #1746] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:53.302 [DEBUG - New I/O worker #1686] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.303 [DEBUG - New I/O worker #1580] (Negotiator.java:764) Authenticated connection [id: 0x4e5a865f, /127.1.193.125:50691 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.305505 4887 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.274195 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46619 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.275322 (+ 1127us) server_negotiation.cc:184] Beginning negotiation 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.275348 (+ 26us) server_negotiation.cc:373] Waiting for connection header 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.275393 (+ 45us) server_negotiation.cc:381] Connection header received 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.275597 (+ 204us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.275626 (+ 29us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.278467 (+ 2841us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.278643 (+ 176us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.295433 (+ 16790us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.296064 (+ 631us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.298048 (+ 1984us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.300695 (+ 2647us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.301515 (+ 820us) server_negotiation.cc:584] Negotiated auth-only TLSv1 with cipher AES256-SHA SSLv3 Kx=RSA Au=RSA Enc=AES(256) Mac=SHA1 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303053 (+ 1538us) server_negotiation.cc:337] Received SASL_INITIATE NegotiatePB request 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303112 (+ 59us) server_negotiation.cc:746] Received SASL_INITIATE request from client 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303141 (+ 29us) server_negotiation.cc:757] Client requested to use mechanism: PLAIN 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303206 (+ 65us) server_negotiation.cc:786] Calling sasl_server_start() 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303286 (+ 80us) server_negotiation.cc:945] Received PLAIN auth, user=jenkins 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303361 (+ 75us) server_negotiation.cc:349] Sending SASL_SUCCESS NegotiatePB response 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.303948 (+ 587us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46619: BlockingWrite error: write error: Broken pipe (error 32) 01:26:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":277} 01:26:53.307 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.308 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=447), Traces: [0ms] refreshing cache from master, [389ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [389ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [389ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [408ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [414ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [447ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.308 [DEBUG - New I/O worker #1580] (Deferred.java:1330) callback=retry RPC@1452586390 returned Deferred@1386949532(state=PENDING, result=null, callback=(continuation of Deferred@397000621 after retry RPC@1452586390), errback=(continuation of Deferred@397000621 after retry RPC@1452586390)), so the following Deferred is getting paused: Deferred@397000621(state=PAUSED, result=Deferred@1386949532, callback=org.apache.kudu.client.AsyncKuduClient$6@553fd994 -> wakeup thread pool-3-thread-36, errback=passthrough -> wakeup thread pool-3-thread-36) 01:26:53.335 [DEBUG - New I/O worker #1646] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.336 [DEBUG - New I/O worker #1925] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.335 [DEBUG - New I/O worker #1741] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.338 [DEBUG - pool-3-thread-10] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.346 [DEBUG - New I/O worker #1646] (Negotiator.java:764) Authenticated connection [id: 0x4703f5ff, /127.1.193.125:50697 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.346 [DEBUG - New I/O worker #1925] (Negotiator.java:764) Authenticated connection [id: 0x55430765, /127.1.193.126:51436 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.348 [DEBUG - New I/O worker #1741] (Negotiator.java:764) Authenticated connection [id: 0xb5260b05, /127.1.193.124:46622 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.350 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.350 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=163), Traces: [0ms] refreshing cache from master, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [163ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.351 [DEBUG - New I/O worker #1646] (Deferred.java:1330) callback=retry RPC@161837393 returned Deferred@228254909(state=PENDING, result=null, callback=(continuation of Deferred@1217952618 after retry RPC@161837393), errback=(continuation of Deferred@1217952618 after retry RPC@161837393)), so the following Deferred is getting paused: Deferred@1217952618(state=PAUSED, result=Deferred@228254909, callback=org.apache.kudu.client.AsyncKuduClient$6@5b979953 -> wakeup thread pool-3-thread-43, errback=passthrough -> wakeup thread pool-3-thread-43) 01:26:53.353 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.355 [DEBUG - New I/O worker #1733] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.355 [DEBUG - New I/O worker #1585] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.358 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.359 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.359 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.359 [DEBUG - New I/O worker #1646] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.359 [DEBUG - New I/O worker #1858] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.359 [DEBUG - New I/O worker #1646] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.361 [DEBUG - pool-3-thread-43] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.361 [DEBUG - New I/O worker #1925] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.361 [DEBUG - New I/O worker #1646] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.365 [DEBUG - New I/O worker #1585] (Negotiator.java:764) Authenticated connection [id: 0xf1652d6a, /127.1.193.125:50700 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.366 [DEBUG - New I/O worker #1741] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.367 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.368 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=187), Traces: [0ms] refreshing cache from master, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [173ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.368 [DEBUG - New I/O worker #1585] (Deferred.java:1330) callback=retry RPC@904717526 returned Deferred@835942067(state=PENDING, result=null, callback=(continuation of Deferred@1959915245 after retry RPC@904717526), errback=(continuation of Deferred@1959915245 after retry RPC@904717526)), so the following Deferred is getting paused: Deferred@1959915245(state=PAUSED, result=Deferred@835942067, callback=org.apache.kudu.client.AsyncKuduClient$6@177730b1 -> wakeup thread pool-3-thread-12, errback=passthrough -> wakeup thread pool-3-thread-12) 01:26:53.371 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.374 [DEBUG - pool-3-thread-34] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.375 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.375 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.376 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.376 [DEBUG - New I/O worker #1585] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.376 [DEBUG - New I/O worker #1585] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.376 [DEBUG - New I/O worker #1733] (Negotiator.java:764) Authenticated connection [id: 0xf15b6f9a, /127.1.193.124:46625 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.377 [DEBUG - New I/O worker #1733] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.377 [INFO - New I/O worker #1733] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:53.377 [DEBUG - New I/O worker #1585] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.378 [INFO - New I/O worker #1858] (Connection.java:435) [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:53.378 [DEBUG - New I/O worker #1858] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:53.379 [INFO - New I/O worker #1858] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:53.377 [DEBUG - pool-3-thread-12] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.380455 4888 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.354028 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51439 01:26:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.358933 (+ 4905us) server_negotiation.cc:184] Beginning negotiation 01:26:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.358964 (+ 31us) server_negotiation.cc:373] Waiting for connection header 01:26:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.359015 (+ 51us) server_negotiation.cc:381] Connection header received 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.359150 (+ 135us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:53.381 [DEBUG - pool-3-thread-53] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.359180 (+ 30us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.359340 (+ 160us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.361259 (+ 1919us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.361564 (+ 305us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.371486 (+ 9922us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.378239 (+ 6753us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51439: BlockingRecv error: recv got EOF from 127.1.193.126:51439 (error 108) 01:26:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":4622} 01:26:53.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.386088 1951 connection.cc:554] Connection torn down before Call kudu.master.MasterService.ConnectToMaster from 127.1.193.124:46625 (request call id 0) could send its response 01:26:53.386 [DEBUG - pool-3-thread-71] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.387 [DEBUG - New I/O worker #1648] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.387 [DEBUG - New I/O worker #1842] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.389 [DEBUG - New I/O worker #1843] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.393 [DEBUG - New I/O worker #1648] (Negotiator.java:764) Authenticated connection [id: 0x0df75ee9, /127.1.193.125:50703 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.395 [DEBUG - New I/O worker #1842] (Negotiator.java:764) Authenticated connection [id: 0xa833aa43, /127.1.193.124:46628 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.396 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.396 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=219), Traces: [0ms] refreshing cache from master, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [219ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.397 [DEBUG - New I/O worker #1648] (Deferred.java:1330) callback=retry RPC@1303566957 returned Deferred@1233999619(state=PENDING, result=null, callback=(continuation of Deferred@210751574 after retry RPC@1303566957), errback=(continuation of Deferred@210751574 after retry RPC@1303566957)), so the following Deferred is getting paused: Deferred@210751574(state=PAUSED, result=Deferred@1233999619, callback=org.apache.kudu.client.AsyncKuduClient$6@6673e310 -> wakeup thread pool-3-thread-65, errback=passthrough -> wakeup thread pool-3-thread-65) 01:26:53.399 [DEBUG - New I/O worker #1843] (Negotiator.java:764) Authenticated connection [id: 0xa56cd3f0, /127.1.193.126:51442 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.400 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.404 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.405 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.405 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.406 [DEBUG - New I/O worker #1648] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.406 [DEBUG - New I/O worker #1648] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.407 [DEBUG - pool-3-thread-65] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.407 [DEBUG - New I/O worker #1842] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.408 [DEBUG - New I/O worker #1843] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.409 [DEBUG - New I/O worker #1648] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.420 [DEBUG - pool-3-thread-56] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.421 [DEBUG - pool-3-thread-20] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.426 [DEBUG - New I/O worker #1649] (Negotiator.java:764) Authenticated connection [id: 0x261e7f58, /127.1.193.125:50673 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.428 [DEBUG - New I/O worker #1651] (Negotiator.java:764) Authenticated connection [id: 0x36d35800, /127.1.193.124:46598 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.432 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.432 [DEBUG - New I/O worker #1830] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.432 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=410), Traces: [0ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [144ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [410ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [410ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.433 [DEBUG - New I/O worker #1649] (Deferred.java:1330) callback=retry RPC@984341693 returned Deferred@1049906992(state=PENDING, result=null, callback=(continuation of Deferred@2073482886 after retry RPC@984341693), errback=(continuation of Deferred@2073482886 after retry RPC@984341693)), so the following Deferred is getting paused: Deferred@2073482886(state=PAUSED, result=Deferred@1049906992, callback=org.apache.kudu.client.AsyncKuduClient$6@2e8da390 -> wakeup thread pool-3-thread-72, errback=passthrough -> wakeup thread pool-3-thread-72) 01:26:53.435 [DEBUG - New I/O worker #1650] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.437 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.442 [DEBUG - New I/O worker #1650] (Negotiator.java:764) Authenticated connection [id: 0xdbbea34c, /127.1.193.125:50706 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.445 [DEBUG - New I/O worker #1829] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.447 [DEBUG - New I/O worker #1830] (Negotiator.java:764) Authenticated connection [id: 0x8004908b, /127.1.193.126:51445 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.450 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.451 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.451 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.451 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=31), Traces: [0ms] refreshing cache from master, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [30ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [30ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.451 [DEBUG - New I/O worker #1829] (Negotiator.java:764) Authenticated connection [id: 0xbf490863, /127.1.193.124:46631 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.452 [DEBUG - New I/O worker #1650] (Deferred.java:1330) callback=retry RPC@447741299 returned Deferred@512781054(state=PENDING, result=null, callback=(continuation of Deferred@1536358216 after retry RPC@447741299), errback=(continuation of Deferred@1536358216 after retry RPC@447741299)), so the following Deferred is getting paused: Deferred@1536358216(state=PAUSED, result=Deferred@512781054, callback=org.apache.kudu.client.AsyncKuduClient$6@fa2f9c5 -> wakeup thread pool-3-thread-56, errback=passthrough -> wakeup thread pool-3-thread-56) 01:26:53.451 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.452 [DEBUG - New I/O worker #1649] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.453 [DEBUG - New I/O worker #1649] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.453 [DEBUG - pool-3-thread-28] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.453 [DEBUG - pool-3-thread-72] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.454 [DEBUG - New I/O worker #1649] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.454 [DEBUG - New I/O worker #1656] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.454 [DEBUG - New I/O worker #1651] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.456 [DEBUG - pool-3-thread-33] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.456 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.460 [DEBUG - New I/O worker #1710] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.460 [DEBUG - New I/O worker #1697] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.461 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.461 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.462 [DEBUG - New I/O worker #1709] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.462 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.471 [DEBUG - New I/O worker #1650] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.472 [DEBUG - New I/O worker #1650] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.468 [DEBUG - pool-3-thread-13] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.468 [DEBUG - New I/O worker #1697] (Negotiator.java:764) Authenticated connection [id: 0x42152186, /127.1.193.125:50709 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.473 [DEBUG - New I/O worker #1829] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.473 [DEBUG - New I/O worker #1830] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.465 [DEBUG - New I/O worker #1710] (Negotiator.java:764) Authenticated connection [id: 0xad632d09, /127.1.193.126:51448 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.472 [DEBUG - pool-3-thread-56] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.473 [DEBUG - New I/O worker #1650] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.478 [DEBUG - New I/O worker #1709] (Negotiator.java:764) Authenticated connection [id: 0xbeef0422, /127.1.193.124:46634 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.479 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.480 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=398), Traces: [0ms] refreshing cache from master, [375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [394ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [397ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.480 [DEBUG - New I/O worker #1697] (Deferred.java:1330) callback=retry RPC@1735413563 returned Deferred@1666172685(state=PENDING, result=null, callback=(continuation of Deferred@642598144 after retry RPC@1735413563), errback=(continuation of Deferred@642598144 after retry RPC@1735413563)), so the following Deferred is getting paused: Deferred@642598144(state=PAUSED, result=Deferred@1666172685, callback=org.apache.kudu.client.AsyncKuduClient$6@72cdd1f0 -> wakeup thread pool-3-thread-76, errback=passthrough -> wakeup thread pool-3-thread-76) 01:26:53.482 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.483 [DEBUG - Time-limited test] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.486 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.486 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.487 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.487 [DEBUG - New I/O worker #1697] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.487 [DEBUG - New I/O worker #1697] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.488 [DEBUG - New I/O worker #1673] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.488 [DEBUG - New I/O worker #1794] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.489 [DEBUG - pool-3-thread-76] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.489 [DEBUG - New I/O worker #1709] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.489 [DEBUG - New I/O worker #1710] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.490 [DEBUG - New I/O worker #1793] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.491 [DEBUG - New I/O worker #1716] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.492 [DEBUG - New I/O worker #1631] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.493 [DEBUG - New I/O worker #1726] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.491 [DEBUG - New I/O worker #1697] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.496 [DEBUG - New I/O worker #1658] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.496 [DEBUG - New I/O worker #1719] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.493 [DEBUG - New I/O worker #1827] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.502 [DEBUG - New I/O worker #1793] (Negotiator.java:764) Authenticated connection [id: 0xe7eb71c4, /127.1.193.126:51454 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.505 [DEBUG - New I/O worker #1631] (Negotiator.java:764) Authenticated connection [id: 0x5fe37058, /127.1.193.125:50715 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.506 [DEBUG - New I/O worker #1794] (Negotiator.java:764) Authenticated connection [id: 0x259c3a94, /127.1.193.126:51451 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.508 [DEBUG - New I/O worker #1673] (Negotiator.java:764) Authenticated connection [id: 0x0c87ef39, /127.1.193.125:50712 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.508 [DEBUG - New I/O worker #1716] (Negotiator.java:764) Authenticated connection [id: 0x65807f99, /127.1.193.124:46637 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.508 [DEBUG - New I/O worker #1719] (Negotiator.java:764) Authenticated connection [id: 0xa6183014, /127.1.193.124:46643 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.509 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.511 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=54), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [52ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [53ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.510 [DEBUG - New I/O worker #1658] (Negotiator.java:764) Authenticated connection [id: 0x97d43367, /127.1.193.125:50718 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.512 [DEBUG - New I/O worker #1827] (Negotiator.java:764) Authenticated connection [id: 0xc2437091, /127.1.193.126:51457 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.514 [DEBUG - New I/O worker #1726] (Negotiator.java:764) Authenticated connection [id: 0xfff3f1b3, /127.1.193.124:46640 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.514 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.515 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=93), Traces: [0ms] refreshing cache from master, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [68ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [91ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.515 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.515 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=61), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [60ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.516 [DEBUG - New I/O worker #1673] (Deferred.java:1330) callback=retry RPC@1875745840 returned Deferred@1811027217(state=PENDING, result=null, callback=(continuation of Deferred@787518987 after retry RPC@1875745840), errback=(continuation of Deferred@787518987 after retry RPC@1875745840)), so the following Deferred is getting paused: Deferred@787518987(state=PAUSED, result=Deferred@1811027217, callback=org.apache.kudu.client.AsyncKuduClient$6@1cbb1cc2 -> wakeup thread pool-3-thread-28, errback=passthrough -> wakeup thread pool-3-thread-28) 01:26:53.511 [DEBUG - New I/O worker #1631] (Deferred.java:1330) callback=retry RPC@576970651 returned Deferred@643587855(state=PENDING, result=null, callback=(continuation of Deferred@1667159456 after retry RPC@576970651), errback=(continuation of Deferred@1667159456 after retry RPC@576970651)), so the following Deferred is getting paused: Deferred@1667159456(state=PAUSED, result=Deferred@643587855, callback=org.apache.kudu.client.AsyncKuduClient$6@6ed41278 -> wakeup thread pool-3-thread-33, errback=passthrough -> wakeup thread pool-3-thread-33) 01:26:53.517 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.518 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.518 [DEBUG - pool-3-thread-40] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.518 [DEBUG - pool-3-thread-60] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.515 [DEBUG - New I/O worker #1658] (Deferred.java:1330) callback=retry RPC@1233699822 returned Deferred@1303858695(state=PENDING, result=null, callback=(continuation of Deferred@146131413 after retry RPC@1233699822), errback=(continuation of Deferred@146131413 after retry RPC@1233699822)), so the following Deferred is getting paused: Deferred@146131413(state=PAUSED, result=Deferred@1303858695, callback=org.apache.kudu.client.AsyncKuduClient$6@366f3129 -> wakeup thread pool-3-thread-20, errback=passthrough -> wakeup thread pool-3-thread-20) 01:26:53.521 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.524 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.524 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.525 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.525 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.525 [DEBUG - New I/O worker #1631] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.526 [DEBUG - New I/O worker #1631] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.525 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.606 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.607 [DEBUG - New I/O worker #1673] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.607 [DEBUG - New I/O worker #1673] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.608 [DEBUG - New I/O worker #1631] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.558 [DEBUG - New I/O worker #1854] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.609 [DEBUG - pool-3-thread-33] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.553 [DEBUG - New I/O worker #1600] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.609 [DEBUG - New I/O worker #1793] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.553 [DEBUG - New I/O worker #1589] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.553 [DEBUG - New I/O worker #1763] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.610 [DEBUG - New I/O worker #1716] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.610 [DEBUG - New I/O worker #1794] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.552 [DEBUG - New I/O worker #1731] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.552 [DEBUG - New I/O worker #1857] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.546 [DEBUG - New I/O worker #1611] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.546 [DEBUG - New I/O worker #1616] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.546 [DEBUG - New I/O worker #1614] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.545 [DEBUG - pool-3-thread-39] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.538 [DEBUG - pool-3-thread-30] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.538 [DEBUG - pool-3-thread-31] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.615 [DEBUG - pool-3-thread-19] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.534 [DEBUG - New I/O worker #1712] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.533 [DEBUG - New I/O worker #1695] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.532 [DEBUG - New I/O worker #1690] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.528 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.618 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.618 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.618 [DEBUG - New I/O worker #1658] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.618 [DEBUG - New I/O worker #1658] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.611 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.609 [DEBUG - New I/O worker #1673] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.619 [DEBUG - New I/O worker #1719] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.609 [DEBUG - New I/O worker #1726] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.619 [DEBUG - pool-3-thread-77] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.623 [DEBUG - pool-3-thread-28] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.624 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.624 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.624 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.624 [DEBUG - New I/O worker #1580] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.625 [DEBUG - New I/O worker #1580] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.626 [DEBUG - pool-3-thread-36] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.626 [DEBUG - New I/O worker #1580] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.627 [DEBUG - pool-3-thread-32] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.630 [DEBUG - New I/O worker #1658] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.632 [DEBUG - New I/O worker #1638] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.633 [DEBUG - New I/O worker #1723] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.633 [DEBUG - pool-3-thread-25] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.634 [DEBUG - pool-3-thread-27] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.634 [DEBUG - pool-3-thread-20] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.635 [DEBUG - New I/O worker #1854] (Negotiator.java:764) Authenticated connection [id: 0xdaa518c7, /127.1.193.126:51469 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.645 [DEBUG - New I/O worker #1976] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.646 [DEBUG - New I/O worker #1740] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.649 [DEBUG - New I/O worker #1581] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.653 [DEBUG - New I/O worker #1644] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.654 [DEBUG - New I/O worker #1763] (Negotiator.java:764) Authenticated connection [id: 0x5df225d4, /127.1.193.124:46653 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.655 [DEBUG - New I/O worker #1600] (Negotiator.java:764) Authenticated connection [id: 0x36875a96, /127.1.193.125:50728 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.656 [DEBUG - New I/O worker #1589] (Negotiator.java:764) Authenticated connection [id: 0xcd7cb399, /127.1.193.125:50727 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.657 [DEBUG - New I/O worker #1731] (Negotiator.java:764) Authenticated connection [id: 0x7d514d4d, /127.1.193.124:46654 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.659 [DEBUG - New I/O worker #1867] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.680 [DEBUG - New I/O worker #1583] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.684 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.684 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=303), Traces: [0ms] refreshing cache from master, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [280ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [287ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.684 [DEBUG - New I/O worker #1589] (Deferred.java:1330) callback=retry RPC@515205892 returned Deferred@445312010(state=PENDING, result=null, callback=(continuation of Deferred@1602773311 after retry RPC@515205892), errback=(continuation of Deferred@1602773311 after retry RPC@515205892)), so the following Deferred is getting paused: Deferred@1602773311(state=PAUSED, result=Deferred@445312010, callback=org.apache.kudu.client.AsyncKuduClient$6@5bc574e3 -> wakeup thread pool-3-thread-53, errback=passthrough -> wakeup thread pool-3-thread-53) 01:26:53.685 [DEBUG - New I/O worker #1596] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.687 [DEBUG - New I/O worker #1723] (Negotiator.java:764) Authenticated connection [id: 0xc8f4aee1, /127.1.193.124:46658 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.688 [DEBUG - New I/O worker #1611] (Negotiator.java:764) Authenticated connection [id: 0x3f140928, /127.1.193.125:50724 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.689 [DEBUG - New I/O worker #1616] (Negotiator.java:764) Authenticated connection [id: 0x6a22dcc8, /127.1.193.126:51463 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.690 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.693 [DEBUG - New I/O worker #1976] (Negotiator.java:764) Authenticated connection [id: 0x635d51d3, /127.1.193.126:51472 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.694 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.694 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.695 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.695 [DEBUG - New I/O worker #1589] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.695 [DEBUG - New I/O worker #1589] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.696 [DEBUG - New I/O worker #1763] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.697 [DEBUG - New I/O worker #1589] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.696 [DEBUG - pool-3-thread-53] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.698792 5284 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.550699 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51468 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551390 (+ 691us) server_negotiation.cc:184] Beginning negotiation 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551417 (+ 27us) server_negotiation.cc:373] Waiting for connection header 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551463 (+ 46us) server_negotiation.cc:381] Connection header received 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551587 (+ 124us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:53.700 [DEBUG - New I/O worker #1732] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551617 (+ 30us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:53.700 [DEBUG - New I/O worker #1613] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551778 (+ 161us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.551982 (+ 204us) server_negotiation.cc:205] Negotiated authn=SASL 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.617535 (+ 65553us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.618122 (+ 587us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.633700 (+ 15578us) server_negotiation.cc:337] Received TLS_HANDSHAKE NegotiatePB request 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.634953 (+ 1253us) server_negotiation.cc:349] Sending TLS_HANDSHAKE NegotiatePB response 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.635175 (+ 222us) server_negotiation.cc:584] Negotiated auth-only TLSv1 with cipher AES256-SHA SSLv3 Kx=RSA Au=RSA Enc=AES(256) Mac=SHA1 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.697484 (+ 62309us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51468: BlockingRecv error: recv got EOF from 127.1.193.126:51468 (error 108) 01:26:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":403} 01:26:53.700 [DEBUG - New I/O worker #1952] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.706 [DEBUG - New I/O worker #1847] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.707 [DEBUG - New I/O worker #1725] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.707 [DEBUG - New I/O worker #1722] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.707 [DEBUG - New I/O worker #1706] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.706 [DEBUG - New I/O worker #1868] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.706 [DEBUG - New I/O worker #1713] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.706 [DEBUG - New I/O worker #1997] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.707 [DEBUG - New I/O worker #1734] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.711 [DEBUG - New I/O worker #1952] (Negotiator.java:764) Authenticated connection [id: 0xdc5c0b3d, /127.1.193.126:51487 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.717 [DEBUG - New I/O worker #1827] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.717 [DEBUG - New I/O worker #1596] (Negotiator.java:764) Authenticated connection [id: 0xee535e46, /127.1.193.125:50748 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.718 [DEBUG - New I/O worker #1693] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.719 [INFO - New I/O worker #1857] (Connection.java:435) [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:53.719 [DEBUG - New I/O worker #1857] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:53.719 [INFO - New I/O worker #1857] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: [peer master-127.1.193.126:40739(127.1.193.126:40739)] disconnected from peer 01:26:53.720 [DEBUG - New I/O worker #1742] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.719 [DEBUG - New I/O worker #1614] (Negotiator.java:764) Authenticated connection [id: 0x9a4e1cb5, /127.1.193.124:46649 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.720 [DEBUG - New I/O worker #1647] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.726 [DEBUG - New I/O worker #1867] (Negotiator.java:764) Authenticated connection [id: 0xd6fec732, /127.1.193.126:51475 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.729 [DEBUG - New I/O worker #1742] (Negotiator.java:764) Authenticated connection [id: 0x22afacc2, /127.1.193.124:46670 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.731 [DEBUG - New I/O worker #1997] (Negotiator.java:764) Authenticated connection [id: 0x56661fa9, /127.1.193.126:51478 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.734 [DEBUG - New I/O worker #1740] (Negotiator.java:764) Authenticated connection [id: 0xa0130b83, /127.1.193.124:46661 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.735 [DEBUG - New I/O worker #1712] (Negotiator.java:764) Authenticated connection [id: 0xa1b9f70c, /127.1.193.126:51460 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.736 [DEBUG - New I/O worker #1690] (Negotiator.java:764) Authenticated connection [id: 0x8ffdacae, /127.1.193.125:50721 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.738 [DEBUG - New I/O worker #1852] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.739 [DEBUG - New I/O worker #1693] (Negotiator.java:764) Authenticated connection [id: 0xf8d20709, /127.1.193.125:50739 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.739 [DEBUG - New I/O worker #1713] (Negotiator.java:764) Authenticated connection [id: 0xffc0cdd7, /127.1.193.124:46664 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.741 [DEBUG - New I/O worker #1613] (Negotiator.java:764) Authenticated connection [id: 0x43510561, /127.1.193.125:50751 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.745 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.745 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=117), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [116ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.745 [DEBUG - New I/O worker #1852] (Negotiator.java:764) Authenticated connection [id: 0x534c762f, /127.1.193.124:46691 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.746 [DEBUG - New I/O worker #1613] (Deferred.java:1330) callback=retry RPC@466620520 returned Deferred@535791562(state=PENDING, result=null, callback=(continuation of Deferred@1525488211 after retry RPC@466620520), errback=(continuation of Deferred@1525488211 after retry RPC@466620520)), so the following Deferred is getting paused: Deferred@1525488211(state=PAUSED, result=Deferred@535791562, callback=org.apache.kudu.client.AsyncKuduClient$6@70354f88 -> wakeup thread pool-3-thread-77, errback=passthrough -> wakeup thread pool-3-thread-77) 01:26:53.745 [DEBUG - New I/O worker #1644] (Negotiator.java:764) Authenticated connection [id: 0xc580d097, /127.1.193.125:50736 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.745 [DEBUG - New I/O worker #1847] (Negotiator.java:764) Authenticated connection [id: 0xe93ca3a4, /127.1.193.126:51493 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.747 [DEBUG - New I/O worker #1706] (Negotiator.java:764) Authenticated connection [id: 0x1212727a, /127.1.193.125:50742 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.747 [DEBUG - New I/O worker #1732] (Negotiator.java:764) Authenticated connection [id: 0x587872f2, /127.1.193.124:46673 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.748 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.749 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.749 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=410), Traces: [0ms] refreshing cache from master, [277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [392ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [402ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [411ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.750 [DEBUG - New I/O worker #1644] (Deferred.java:1330) callback=retry RPC@1559604361 returned Deferred@1489642447(state=PENDING, result=null, callback=(continuation of Deferred@499689138 after retry RPC@1559604361), errback=(continuation of Deferred@499689138 after retry RPC@1559604361)), so the following Deferred is getting paused: Deferred@499689138(state=PAUSED, result=Deferred@1489642447, callback=org.apache.kudu.client.AsyncKuduClient$6@5dff6204 -> wakeup thread pool-3-thread-10, errback=passthrough -> wakeup thread pool-3-thread-10) 01:26:53.750 [DEBUG - New I/O worker #1632] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.751 [DEBUG - New I/O worker #1579] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.753 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.753 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.753 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.753 [DEBUG - New I/O worker #1613] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.754 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.754 [DEBUG - New I/O worker #1613] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.755 [DEBUG - pool-3-thread-77] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.755 [DEBUG - New I/O worker #1860] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state NEGOTIATING due to: connection disconnected 01:26:53.755 [DEBUG - New I/O worker #1613] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.755 [DEBUG - New I/O worker #1736] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state NEGOTIATING due to: connection disconnected 01:26:53.756 [DEBUG - New I/O worker #1609] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.755 [INFO - New I/O worker #1860] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.126:40739: connection disconnected 01:26:53.756 [INFO - New I/O worker #1736] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:53.762 [DEBUG - New I/O worker #1725] (Negotiator.java:764) Authenticated connection [id: 0x21a74c3f, /127.1.193.124:46679 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.764 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.764 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.764 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.764 [DEBUG - New I/O worker #1644] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.765 [DEBUG - New I/O worker #1644] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.765 [DEBUG - pool-3-thread-10] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.765 [DEBUG - New I/O worker #1867] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.765 [DEBUG - New I/O worker #1644] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.766 [DEBUG - New I/O worker #1740] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.770 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.771 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=137), Traces: [0ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [83ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [137ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.771 [DEBUG - New I/O worker #1596] (Deferred.java:1330) callback=retry RPC@215679666 returned Deferred@149194737(state=PENDING, result=null, callback=(continuation of Deferred@1306921097 after retry RPC@215679666), errback=(continuation of Deferred@1306921097 after retry RPC@215679666)), so the following Deferred is getting paused: Deferred@1306921097(state=PAUSED, result=Deferred@149194737, callback=org.apache.kudu.client.AsyncKuduClient$6@28fb9d50 -> wakeup thread pool-3-thread-25, errback=passthrough -> wakeup thread pool-3-thread-25) 01:26:53.772 [DEBUG - New I/O worker #1772] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.776043 4567 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774087 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.124:46676 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774417 (+ 330us) server_negotiation.cc:184] Beginning negotiation 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774442 (+ 25us) server_negotiation.cc:373] Waiting for connection header 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774487 (+ 45us) server_negotiation.cc:381] Connection header received 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774607 (+ 120us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774639 (+ 32us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.774930 (+ 291us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:53.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.775237 (+ 307us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.124:46676: BlockingWrite error: write error: Broken pipe (error 32) 01:26:53.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":118} 01:26:53.778 [DEBUG - New I/O worker #1730] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.780 [DEBUG - New I/O worker #1966] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.781 [DEBUG - New I/O worker #1959] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.782 [DEBUG - New I/O worker #1859] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.782 [DEBUG - New I/O worker #1773] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.783846 4888 negotiation.cc:313] Failed RPC negotiation. Trace: 01:26:53.785 [DEBUG - New I/O worker #1579] (Negotiator.java:764) Authenticated connection [id: 0xc241b82c, /127.1.193.125:50757 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.695970 (+ 0us) reactor.cc:583] Submitting negotiation task for server connection from 127.1.193.126:51490 01:26:53.785 [DEBUG - New I/O worker #1844] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.714318 (+ 18348us) server_negotiation.cc:184] Beginning negotiation 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.714345 (+ 27us) server_negotiation.cc:373] Waiting for connection header 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.714388 (+ 43us) server_negotiation.cc:381] Connection header received 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.714502 (+ 114us) server_negotiation.cc:337] Received NEGOTIATE NegotiatePB request 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.714527 (+ 25us) server_negotiation.cc:420] Received NEGOTIATE request from client 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.782399 (+ 67872us) spinlock_profiling.cc:243] Waited 67.6 ms on lock 0x7b4400001fa8. stack: 00007f4d04f40634 00007f4d04f40530 00007f4d049131c4 00007f4d0dc2c484 00007f4d0dc2c419 00007f4d04f3cc87 00007f4d04f3cb42 00007f4d0dc1b839 00007f4d0dc1b7c8 00007f4d06816cde 00007f4d06811379 00007f4d0680fdd8 00007f4d067d9d8b 00007f4d067d9581 00007f4d067f280d 00007f4d067f26af 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.782517 (+ 118us) server_negotiation.cc:349] Sending NEGOTIATE NegotiatePB response 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 0219 01:26:53.782928 (+ 411us) negotiation.cc:304] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.193.126:51490: BlockingWrite error: write error: Broken pipe (error 32) 01:26:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Metrics: {"server-negotiator.queue_time_us":18127,"spinlock_wait_cycles":162242304} 01:26:53.787 [DEBUG - New I/O worker #1772] (Negotiator.java:764) Authenticated connection [id: 0xcbbe5b22, /127.1.193.124:46667 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.793 [DEBUG - New I/O worker #1730] (Negotiator.java:764) Authenticated connection [id: 0xf2a34329, /127.1.193.124:46688 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.793 [DEBUG - New I/O worker #1959] (Negotiator.java:764) Authenticated connection [id: 0xe80cbb76, /127.1.193.126:51505 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.798 [DEBUG - New I/O worker #1844] (Negotiator.java:764) Authenticated connection [id: 0x59e2686f, /127.1.193.126:51499 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.799 [DEBUG - New I/O worker #1632] (Negotiator.java:764) Authenticated connection [id: 0x9715b0e2, /127.1.193.125:50754 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.800 [DEBUG - New I/O worker #1966] (Negotiator.java:764) Authenticated connection [id: 0x546b2bd8, /127.1.193.126:51502 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.800 [DEBUG - New I/O worker #1734] (Negotiator.java:764) Authenticated connection [id: 0x6f338345, /127.1.193.124:46682 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.801 [DEBUG - New I/O worker #1603] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.802 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.803 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=415), Traces: [0ms] refreshing cache from master, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [338ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [375ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [415ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.803 [DEBUG - New I/O worker #1611] (Deferred.java:1330) callback=retry RPC@62698181 returned Deferred@126039658(state=PENDING, result=null, callback=(continuation of Deferred@1115797758 after retry RPC@62698181), errback=(continuation of Deferred@1115797758 after retry RPC@62698181)), so the following Deferred is getting paused: Deferred@1115797758(state=PAUSED, result=Deferred@126039658, callback=org.apache.kudu.client.AsyncKuduClient$6@1545ae39 -> wakeup thread pool-3-thread-71, errback=passthrough -> wakeup thread pool-3-thread-71) 01:26:53.806 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.806 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.806 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=179), Traces: [0ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [138ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [138ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [179ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.807 [DEBUG - New I/O worker #1632] (Deferred.java:1330) callback=retry RPC@853386886 returned Deferred@920790021(state=PENDING, result=null, callback=(continuation of Deferred@1944104125 after retry RPC@853386886), errback=(continuation of Deferred@1944104125 after retry RPC@853386886)), so the following Deferred is getting paused: Deferred@1944104125(state=PAUSED, result=Deferred@920790021, callback=org.apache.kudu.client.AsyncKuduClient$6@1def2453 -> wakeup thread pool-3-thread-32, errback=passthrough -> wakeup thread pool-3-thread-32) 01:26:53.807 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.807 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=192), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [192ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.808 [DEBUG - New I/O worker #1579] (Deferred.java:1330) callback=retry RPC@1598375506 returned Deferred@1534777698(state=PENDING, result=null, callback=(continuation of Deferred@511200361 after retry RPC@1598375506), errback=(continuation of Deferred@511200361 after retry RPC@1598375506)), so the following Deferred is getting paused: Deferred@511200361(state=PAUSED, result=Deferred@1534777698, callback=org.apache.kudu.client.AsyncKuduClient$6@5a2e6f96 -> wakeup thread pool-3-thread-19, errback=passthrough -> wakeup thread pool-3-thread-19) 01:26:53.809 [DEBUG - New I/O worker #1868] (Negotiator.java:764) Authenticated connection [id: 0xd0ef50fe, /127.1.193.126:51484 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.810 [DEBUG - New I/O worker #1647] (Negotiator.java:764) Authenticated connection [id: 0x397cb54e, /127.1.193.125:50760 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.812 [DEBUG - New I/O worker #1654] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.813 [DEBUG - New I/O worker #1859] (Negotiator.java:764) Authenticated connection [id: 0xe64f29b5, /127.1.193.126:51496 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.813 [DEBUG - New I/O worker #1609] (Negotiator.java:764) Authenticated connection [id: 0x74251b87, /127.1.193.125:50763 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.814 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.814 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=199), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [186ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [199ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.815 [DEBUG - New I/O worker #1647] (Deferred.java:1330) callback=retry RPC@1300295041 returned Deferred@1237283049(state=PENDING, result=null, callback=(continuation of Deferred@213772218 after retry RPC@1300295041), errback=(continuation of Deferred@213772218 after retry RPC@1300295041)), so the following Deferred is getting paused: Deferred@213772218(state=PAUSED, result=Deferred@1237283049, callback=org.apache.kudu.client.AsyncKuduClient$6@775dbe67 -> wakeup thread pool-3-thread-31, errback=passthrough -> wakeup thread pool-3-thread-31) 01:26:53.817 [DEBUG - New I/O worker #1722] (Negotiator.java:764) Authenticated connection [id: 0x2b2c7160, /127.1.193.124:46685 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.817 [DEBUG - New I/O worker #1603] (Negotiator.java:764) Authenticated connection [id: 0x44cd5edf, /127.1.193.125:50766 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.817 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.821 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.820 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.823 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=449), Traces: [0ms] refreshing cache from master, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [265ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [400ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [447ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.824 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=340), Traces: [0ms] refreshing cache from master, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [287ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [291ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [337ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.824 [DEBUG - New I/O worker #1600] (Deferred.java:1330) callback=retry RPC@1192149124 returned Deferred@1127308477(state=PENDING, result=null, callback=(continuation of Deferred@104057535 after retry RPC@1192149124), errback=(continuation of Deferred@104057535 after retry RPC@1192149124)), so the following Deferred is getting paused: Deferred@104057535(state=PAUSED, result=Deferred@1127308477, callback=org.apache.kudu.client.AsyncKuduClient$6@644f2eb -> wakeup thread pool-3-thread-34, errback=passthrough -> wakeup thread pool-3-thread-34) 01:26:53.824 [DEBUG - New I/O worker #1693] (Deferred.java:1330) callback=retry RPC@1836982667 returned Deferred@1765914876(state=PENDING, result=null, callback=(continuation of Deferred@742599600 after retry RPC@1836982667), errback=(continuation of Deferred@742599600 after retry RPC@1836982667)), so the following Deferred is getting paused: Deferred@742599600(state=PAUSED, result=Deferred@1765914876, callback=org.apache.kudu.client.AsyncKuduClient$6@1cba807 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:26:53.825 [DEBUG - pool-3-thread-74] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.818 [DEBUG - New I/O worker #1] (Connection.java:429) [peer master-127.1.193.125:37435(127.1.193.125:37435)] encountered a read timeout; closing the channel 01:26:53.828 [DEBUG - New I/O worker #1] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: [peer master-127.1.193.125:37435(127.1.193.125:37435)] encountered a read timeout; closing the channel 01:26:53.829 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.836 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.837 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.840 [DEBUG - New I/O worker #1773] (Negotiator.java:764) Authenticated connection [id: 0x46b735c6, /127.1.193.126:51481 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.843 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.843 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.843 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.843 [DEBUG - New I/O worker #1611] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.844 [DEBUG - New I/O worker #1611] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.844 [DEBUG - pool-3-thread-71] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.845 [DEBUG - New I/O worker #1614] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.845 [DEBUG - New I/O worker #1611] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.845 [DEBUG - New I/O worker #1616] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.846 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.846 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.850 [DEBUG - New I/O worker #1654] (Negotiator.java:764) Authenticated connection [id: 0x94ffec8a, /127.1.193.125:50745 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.851 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.851 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.852 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.852 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.852 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.852 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.853 [DEBUG - New I/O worker #1647] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.853 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.852 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.853 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.853 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.854 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=381), Traces: [0ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [320ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [371ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [381ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.854 [DEBUG - New I/O worker #1706] (Deferred.java:1330) callback=retry RPC@2083640328 returned Deferred@2014207830(state=PENDING, result=null, callback=(continuation of Deferred@1024253491 after retry RPC@2083640328), errback=(continuation of Deferred@1024253491 after retry RPC@2083640328)), so the following Deferred is getting paused: Deferred@1024253491(state=PAUSED, result=Deferred@2014207830, callback=org.apache.kudu.client.AsyncKuduClient$6@7e188ab0 -> wakeup thread pool-3-thread-13, errback=passthrough -> wakeup thread pool-3-thread-13) 01:26:53.852 [DEBUG - New I/O worker #1579] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.855 [DEBUG - New I/O worker #1579] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.855 [DEBUG - pool-3-thread-19] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.856 [DEBUG - New I/O worker #1859] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.856 [DEBUG - New I/O worker #1734] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.856 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.854 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.853 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=334), Traces: [0ms] refreshing cache from master, [11ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [11ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [11ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [333ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.857 [DEBUG - New I/O worker #1579] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.857 [DEBUG - New I/O worker #1690] (Deferred.java:1330) callback=retry RPC@1075060930 returned Deferred@1143712792(state=PENDING, result=null, callback=(continuation of Deferred@19475193 after retry RPC@1075060930), errback=(continuation of Deferred@19475193 after retry RPC@1075060930)), so the following Deferred is getting paused: Deferred@19475193(state=PAUSED, result=Deferred@1143712792, callback=org.apache.kudu.client.AsyncKuduClient$6@51914533 -> wakeup thread pool-3-thread-60, errback=passthrough -> wakeup thread pool-3-thread-60) 01:26:53.853 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.853 [DEBUG - New I/O worker #1647] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.858 [DEBUG - pool-3-thread-31] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.858 [DEBUG - New I/O worker #1844] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.859 [DEBUG - New I/O worker #1647] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.860 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.857 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.860 [DEBUG - New I/O worker #1632] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.861 [DEBUG - New I/O worker #1632] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.861 [DEBUG - pool-3-thread-32] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.862 [DEBUG - New I/O worker #1725] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.862 [DEBUG - New I/O worker #1847] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.857 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.863 [DEBUG - New I/O worker #1600] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.856 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=222), Traces: [0ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [137ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [179ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.863 [DEBUG - New I/O worker #1654] (Deferred.java:1330) callback=retry RPC@371632837 returned Deferred@303639769(state=PENDING, result=null, callback=(continuation of Deferred@1461428478 after retry RPC@371632837), errback=(continuation of Deferred@1461428478 after retry RPC@371632837)), so the following Deferred is getting paused: Deferred@1461428478(state=PAUSED, result=Deferred@303639769, callback=org.apache.kudu.client.AsyncKuduClient$6@455579bb -> wakeup thread pool-3-thread-27, errback=passthrough -> wakeup thread pool-3-thread-27) 01:26:53.863 [DEBUG - New I/O worker #1600] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.858 [DEBUG - New I/O worker #1722] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.864 [DEBUG - pool-3-thread-34] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.864 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.865 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=250), Traces: [0ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [140ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [250ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.865 [DEBUG - New I/O worker #1603] (Deferred.java:1330) callback=retry RPC@975519723 returned Deferred@1041935479(state=PENDING, result=null, callback=(continuation of Deferred@2065184208 after retry RPC@975519723), errback=(continuation of Deferred@2065184208 after retry RPC@975519723)), so the following Deferred is getting paused: Deferred@2065184208(state=PAUSED, result=Deferred@1041935479, callback=org.apache.kudu.client.AsyncKuduClient$6@59e00f4b -> wakeup thread pool-3-thread-39, errback=passthrough -> wakeup thread pool-3-thread-39) 01:26:53.865 [DEBUG - New I/O worker #1731] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.866 [DEBUG - New I/O worker #1854] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.863 [DEBUG - New I/O worker #1632] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.865 [DEBUG - New I/O worker #1600] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.873 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.873 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.873 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=258), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [182ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [189ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [259ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.874 [DEBUG - New I/O worker #1609] (Deferred.java:1330) callback=retry RPC@1720852758 returned Deferred@1655548188(state=PENDING, result=null, callback=(continuation of Deferred@665791277 after retry RPC@1720852758), errback=(continuation of Deferred@665791277 after retry RPC@1720852758)), so the following Deferred is getting paused: Deferred@665791277(state=PAUSED, result=Deferred@1655548188, callback=org.apache.kudu.client.AsyncKuduClient$6@61afcb16 -> wakeup thread pool-3-thread-30, errback=passthrough -> wakeup thread pool-3-thread-30) 01:26:53.874 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.874 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.875 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.875 [DEBUG - New I/O worker #1596] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.874 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.875 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.876 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.875 [DEBUG - New I/O worker #1596] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.876 [DEBUG - New I/O worker #1706] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.877 [DEBUG - pool-3-thread-25] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.877 [DEBUG - New I/O worker #1732] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.877 [DEBUG - New I/O worker #1952] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.877 [DEBUG - New I/O worker #1596] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.877 [DEBUG - New I/O worker #1706] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.878 [DEBUG - pool-3-thread-13] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.878 [DEBUG - New I/O worker #1773] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.878 [DEBUG - New I/O worker #1772] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.879 [DEBUG - New I/O worker #1706] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.882 [DEBUG - pool-3-thread-8] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.883 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.884 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.884 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.884 [DEBUG - New I/O worker #1603] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.885 [DEBUG - New I/O worker #1603] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.885 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.885 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.885 [DEBUG - pool-3-thread-39] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.885 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.886 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.921 [DEBUG - New I/O worker #4] (Connection.java:429) [peer 6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)] encountered a read timeout; closing the channel 01:26:53.922 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.922 [DEBUG - New I/O worker #1693] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.922 [DEBUG - New I/O worker #1693] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.923 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.923 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.916 [DEBUG - New I/O worker #1638] (Negotiator.java:764) Authenticated connection [id: 0x3e84e443, /127.1.193.125:50733 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.914 [DEBUG - New I/O worker #1695] (Negotiator.java:764) Authenticated connection [id: 0x01390850, /127.1.193.124:46646 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.905 [DEBUG - New I/O worker #1973] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.927 [DEBUG - New I/O worker #1603] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.928 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.904 [DEBUG - New I/O worker #1727] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.904 [DEBUG - New I/O worker #1624] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.930 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.931 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=412), Traces: [0ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [173ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [245ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [412ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.931 [DEBUG - New I/O worker #1638] (Deferred.java:1330) callback=retry RPC@1252071045 returned Deferred@1319013129(state=PENDING, result=null, callback=(continuation of Deferred@194780350 after retry RPC@1252071045), errback=(continuation of Deferred@194780350 after retry RPC@1252071045)), so the following Deferred is getting paused: Deferred@194780350(state=PAUSED, result=Deferred@1319013129, callback=org.apache.kudu.client.AsyncKuduClient$6@788a409c -> wakeup thread pool-3-thread-40, errback=passthrough -> wakeup thread pool-3-thread-40) 01:26:53.932 [DEBUG - New I/O worker #1693] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.886 [DEBUG - New I/O worker #1852] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.928 [DEBUG - New I/O worker #1959] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.935 [DEBUG - pool-3-thread-9] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.924 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.936 [DEBUG - New I/O worker #1690] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.936 [DEBUG - New I/O worker #1690] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.924 [DEBUG - New I/O worker #1997] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.923 [DEBUG - New I/O worker #1713] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.923 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.937 [DEBUG - New I/O worker #1712] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.923 [DEBUG - Time-limited test] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.939 [DEBUG - New I/O worker #1624] (Negotiator.java:764) Authenticated connection [id: 0x066dce02, /127.1.193.125:50769 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.922 [DEBUG - New I/O worker #4] (Connection.java:688) [peer 6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)] cleaning up while in state READY due to: [peer 6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)] encountered a read timeout; closing the channel 01:26:53.939 [DEBUG - New I/O worker #1695] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.938 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.938 [DEBUG - New I/O worker #1973] (Negotiator.java:764) Authenticated connection [id: 0x11dff2d6, /127.1.193.126:51508 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.938 [DEBUG - New I/O worker #1690] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.937 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.937 [DEBUG - pool-3-thread-60] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.936 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.943 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.940 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.944 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.944 [DEBUG - New I/O worker #1609] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.944 [DEBUG - New I/O worker #1654] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.945 [DEBUG - New I/O worker #1609] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.946 [DEBUG - pool-3-thread-30] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.946 [DEBUG - New I/O worker #1966] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.946 [DEBUG - New I/O worker #1609] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.945 [DEBUG - New I/O worker #1654] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.947 [DEBUG - pool-3-thread-27] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.947 [DEBUG - New I/O worker #1742] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.946 [DEBUG - New I/O worker #1730] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.948 [DEBUG - New I/O worker #1868] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.947 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.948 [DEBUG - New I/O worker #1654] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.948 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=122), Traces: [0ms] refreshing cache from master, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [117ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.949 [DEBUG - New I/O worker #1624] (Deferred.java:1330) callback=retry RPC@1796832846 returned Deferred@1864824821(state=PENDING, result=null, callback=(continuation of Deferred@707032181 after retry RPC@1796832846), errback=(continuation of Deferred@707032181 after retry RPC@1796832846)), so the following Deferred is getting paused: Deferred@707032181(state=PAUSED, result=Deferred@1864824821, callback=org.apache.kudu.client.AsyncKuduClient$6@3b0fff00 -> wakeup thread pool-3-thread-74, errback=passthrough -> wakeup thread pool-3-thread-74) 01:26:53.950 [DEBUG - pool-3-thread-48] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.951 [DEBUG - New I/O worker #1727] (Negotiator.java:764) Authenticated connection [id: 0x1eb64500, /127.1.193.124:46694 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.953 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.956 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.956 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.956 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.957 [DEBUG - New I/O worker #1638] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.957 [DEBUG - New I/O worker #1638] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.958 [DEBUG - pool-3-thread-40] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.958 [DEBUG - New I/O worker #1723] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.958 [DEBUG - New I/O worker #1638] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.958 [DEBUG - New I/O worker #1976] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.958 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.959 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.959 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.960 [DEBUG - New I/O worker #1624] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.960 [DEBUG - New I/O worker #1624] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.961 [DEBUG - pool-3-thread-74] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.961 [DEBUG - New I/O worker #1973] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:53.961182 1948 connection.cc:647] server connection from 127.1.193.124:46694 recv error: Network error: recv error from 0.0.0.0:0: Transport endpoint is not connected (error 107) 01:26:53.961 [DEBUG - New I/O worker #1624] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:53.963 [DEBUG - New I/O worker #1727] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.963 [INFO - New I/O worker #1727] (ConnectToCluster.java:361) Unable to connect to master 127.1.193.124:58247: connection disconnected 01:26:53.963 [DEBUG - New I/O worker #1633] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:53.963 [DEBUG - New I/O worker #1845] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:53.968 [DEBUG - New I/O worker #1724] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:53.973 [DEBUG - New I/O worker #1845] (Negotiator.java:764) Authenticated connection [id: 0x3aadaf15, /127.1.193.126:51511 => /127.1.193.126:40739] using SASL/PLAIN 01:26:53.974 [DEBUG - New I/O worker #1633] (Negotiator.java:764) Authenticated connection [id: 0x3613aed4, /127.1.193.125:50772 => /127.1.193.125:37435] using SASL/PLAIN 01:26:53.977 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:53.978 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=95), Traces: [0ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [95ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [95ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:53.978 [DEBUG - New I/O worker #1724] (Negotiator.java:764) Authenticated connection [id: 0xa07dc6cf, /127.1.193.124:46697 => /127.1.193.124:58247] using SASL/PLAIN 01:26:53.979 [DEBUG - pool-3-thread-75] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:26:53.981 [DEBUG - New I/O worker #1633] (Deferred.java:1330) callback=retry RPC@105376296 returned Deferred@41446968(state=PENDING, result=null, callback=(continuation of Deferred@1199235091 after retry RPC@105376296), errback=(continuation of Deferred@1199235091 after retry RPC@105376296)), so the following Deferred is getting paused: Deferred@1199235091(state=PAUSED, result=Deferred@41446968, callback=org.apache.kudu.client.AsyncKuduClient$6@4f5807f1 -> wakeup thread pool-3-thread-8, errback=passthrough -> wakeup thread pool-3-thread-8) 01:26:53.983 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:53.987 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:53.988 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:53.988 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:53.989 [DEBUG - New I/O worker #1633] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:53.989 [DEBUG - New I/O worker #1633] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:53.991 [DEBUG - pool-3-thread-8] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:53.992 [DEBUG - New I/O worker #1724] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:53.993 [DEBUG - New I/O worker #1845] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:53.994 [DEBUG - New I/O worker #1633] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:54.021 [DEBUG - New I/O worker #1659] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:54.021 [DEBUG - New I/O worker #1981] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:54.021 [DEBUG - New I/O worker #1825] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:54.025 [DEBUG - New I/O worker #1755] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:54.025 [DEBUG - New I/O worker #1737] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:54.026 [DEBUG - New I/O worker #1619] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:54.032 [DEBUG - New I/O worker #5] (Connection.java:429) [peer b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)] encountered a read timeout; closing the channel 01:26:54.032 [DEBUG - New I/O worker #5] (Connection.java:688) [peer b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)] cleaning up while in state READY due to: [peer b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)] encountered a read timeout; closing the channel 01:26:54.035 [DEBUG - New I/O worker #1737] (Negotiator.java:764) Authenticated connection [id: 0xba69ef89, /127.1.193.124:46703 => /127.1.193.124:58247] using SASL/PLAIN 01:26:54.036 [DEBUG - New I/O worker #1825] (Negotiator.java:764) Authenticated connection [id: 0xc305ae6e, /127.1.193.124:46700 => /127.1.193.124:58247] using SASL/PLAIN 01:26:54.037 [DEBUG - New I/O worker #1981] (Negotiator.java:764) Authenticated connection [id: 0xfb1c5ab8, /127.1.193.126:51514 => /127.1.193.126:40739] using SASL/PLAIN 01:26:54.037 [DEBUG - New I/O worker #1755] (Negotiator.java:764) Authenticated connection [id: 0xb2258606, /127.1.193.126:51517 => /127.1.193.126:40739] using SASL/PLAIN 01:26:54.041 [DEBUG - New I/O worker #1659] (Negotiator.java:764) Authenticated connection [id: 0x1e6da5e5, /127.1.193.125:50775 => /127.1.193.125:37435] using SASL/PLAIN 01:26:54.044 [DEBUG - New I/O worker #1619] (Negotiator.java:764) Authenticated connection [id: 0xceb2a569, /127.1.193.125:50778 => /127.1.193.125:37435] using SASL/PLAIN 01:26:54.045 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:54.047 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:54.077 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=126), Traces: [0ms] refreshing cache from master, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [92ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [95ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [97ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:54.077 [DEBUG - New I/O worker #1619] (Deferred.java:1330) callback=retry RPC@1437052027 returned Deferred@1368924695(state=PENDING, result=null, callback=(continuation of Deferred@345679424 after retry RPC@1437052027), errback=(continuation of Deferred@345679424 after retry RPC@1437052027)), so the following Deferred is getting paused: Deferred@345679424(state=PAUSED, result=Deferred@1368924695, callback=org.apache.kudu.client.AsyncKuduClient$6@4d68fec5 -> wakeup thread pool-3-thread-48, errback=passthrough -> wakeup thread pool-3-thread-48) 01:26:54.075 [DEBUG - New I/O worker #1720] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.124 01:26:54.073 [DEBUG - New I/O worker #1652] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.125 01:26:54.073 [DEBUG - New I/O worker #1828] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.1.193.126 01:26:54.047 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=67), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [60ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, deferred=null) after lookup 01:26:54.079 [DEBUG - New I/O worker #1659] (Deferred.java:1330) callback=retry RPC@177601444 returned Deferred@246054446(state=PENDING, result=null, callback=(continuation of Deferred@1269363103 after retry RPC@177601444), errback=(continuation of Deferred@1269363103 after retry RPC@177601444)), so the following Deferred is getting paused: Deferred@1269363103(state=PAUSED, result=Deferred@246054446, callback=org.apache.kudu.client.AsyncKuduClient$6@5f44e551 -> wakeup thread pool-3-thread-75, errback=passthrough -> wakeup thread pool-3-thread-75) 01:26:54.081 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:54.081 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:54.086 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:54.086 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:54.086 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:54.087 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:54.087 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:54.087 [DEBUG - New I/O worker #1659] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:54.087 [DEBUG - New I/O worker #1659] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:54.088 [DEBUG - pool-3-thread-75] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:54.088 [DEBUG - New I/O worker #1825] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:54.089 [DEBUG - New I/O worker #1659] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:54.086 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:54.091 [DEBUG - New I/O worker #1619] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:54.091 [DEBUG - New I/O worker #1981] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:54.092 [DEBUG - New I/O worker #1619] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:54.093 [DEBUG - pool-3-thread-48] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:54.093 [DEBUG - New I/O worker #1737] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:54.094 [DEBUG - New I/O worker #1755] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:54.094 [DEBUG - New I/O worker #1619] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:54.095 [DEBUG - New I/O worker #1720] (Negotiator.java:764) Authenticated connection [id: 0xf86f59d1, /127.1.193.124:46706 => /127.1.193.124:58247] using SASL/PLAIN 01:26:54.095 [DEBUG - New I/O worker #1828] (Negotiator.java:764) Authenticated connection [id: 0x91bcd0ab, /127.1.193.126:51520 => /127.1.193.126:40739] using SASL/PLAIN 01:26:54.095 [DEBUG - New I/O worker #1652] (Negotiator.java:764) Authenticated connection [id: 0xe9c3f2f5, /127.1.193.125:50781 => /127.1.193.125:37435] using SASL/PLAIN 01:26:54.098 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:26:54.099 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=GetTableSchema, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=163), Traces: [0ms] refreshing cache from master, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.125:37435, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.124:58247, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.193.126:40739, [163ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.125:37435: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.126:40739: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.1.193.124:58247: OK, deferred=null) after lookup 01:26:54.100 [DEBUG - New I/O worker #1652] (Deferred.java:1330) callback=retry RPC@606859962 returned Deferred@538206951(state=PENDING, result=null, callback=(continuation of Deferred@1696004225 after retry RPC@606859962), errback=(continuation of Deferred@1696004225 after retry RPC@606859962)), so the following Deferred is getting paused: Deferred@1696004225(state=PAUSED, result=Deferred@538206951, callback=org.apache.kudu.client.AsyncKuduClient$6@2ffcf505 -> wakeup thread pool-3-thread-9, errback=passthrough -> wakeup thread pool-3-thread-9) 01:26:54.102 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:761) Opened table 9829e8c5872f49779e6db7a28aaa0015 01:26:54.105 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:2118) Learned about tablet 02d12d6ae86c466fb511f0ddef2f6fc7 for table 'testManyClients' with partition [, 0x8000000A) 01:26:54.106 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:2118) Learned about tablet 71e4dc0a850f4442b6c11bfcaa1b5ce8 for table 'testManyClients' with partition [0x8000000A, 0x80000014) 01:26:54.106 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:2118) Learned about tablet c0626d9f34c84a0488641522977ae3d0 for table 'testManyClients' with partition [0x80000014, 0x8000001E) 01:26:54.107 [DEBUG - New I/O worker #1652] (AsyncKuduClient.java:2118) Learned about tablet 8062ddd6695848d6b53496b0c2a33e88 for table 'testManyClients' with partition [0x8000001E, ) 01:26:54.107 [DEBUG - New I/O worker #1652] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x8000000A, ttl=299999, tablet=02d12d6ae86c466fb511f0ddef2f6fc7@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000000A, upperBoundPartitionKey=0x80000014, ttl=299999, tablet=71e4dc0a850f4442b6c11bfcaa1b5ce8@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533)[L],b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)]}, Tablet{lowerBoundPartitionKey=0x80000014, upperBoundPartitionKey=0x8000001E, ttl=299999, tablet=c0626d9f34c84a0488641522977ae3d0@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}, Tablet{lowerBoundPartitionKey=0x8000001E, upperBoundPartitionKey=0x, ttl=299999, tablet=8062ddd6695848d6b53496b0c2a33e88@[134794f36230463ea5d0f3f800c08596(127.1.193.66:39259),6eaa0da28e264c548dcf48b8ea8f690e(127.1.193.65:51533),b9bbb727f40b43e2956d6f77e8505f0b(127.1.193.67:36179)[L]]}] 01:26:54.108 [DEBUG - pool-3-thread-9] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:54.108 [DEBUG - New I/O worker #1720] (Connection.java:688) [peer master-127.1.193.124:58247(127.1.193.124:58247)] cleaning up while in state READY due to: connection disconnected 01:26:54.108 [DEBUG - New I/O worker #1828] (Connection.java:688) [peer master-127.1.193.126:40739(127.1.193.126:40739)] cleaning up while in state READY due to: connection disconnected 01:26:54.108 [DEBUG - New I/O worker #1652] (Connection.java:688) [peer master-127.1.193.125:37435(127.1.193.125:37435)] cleaning up while in state READY due to: connection disconnected 01:26:54.159 [DEBUG - Test worker] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:26:54.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:54.176067 1797 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 2003 01:26:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.209023 1886 connection.cc:647] server connection from 127.1.193.65:33962 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:33962): Connection reset by peer (error 104) 01:26:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.210170 1949 connection.cc:647] server connection from 127.1.193.65:34799 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:34799): Connection reset by peer (error 104) 01:26:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.211326 1814 connection.cc:647] server connection from 127.1.193.65:36791 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:36791): Connection reset by peer (error 104) 01:26:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.211480 2174 connection.cc:647] client connection to 127.1.193.65:51533 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:51533): Connection reset by peer (error 104) 01:26:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.214278 2171 connection.cc:647] server connection from 127.1.193.65:54130 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:54130): Connection reset by peer (error 104) 01:26:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.214347 2305 connection.cc:647] client connection to 127.1.193.65:51533 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:51533): Connection reset by peer (error 104) 01:26:54.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:54.215689 1797 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 2155 01:26:54.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.215760 2305 connection.cc:647] server connection from 127.1.193.65:46811 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.65:46811): Connection reset by peer (error 104) 01:26:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.262351 1948 connection.cc:647] server connection from 127.1.193.66:46750 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.66:46750): Connection reset by peer (error 104) 01:26:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.262531 1878 connection.cc:647] server connection from 127.1.193.66:57484 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.66:57484): Connection reset by peer (error 104) 01:26:54.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.263667 1814 connection.cc:647] server connection from 127.1.193.66:38434 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.66:38434): Connection reset by peer (error 104) 01:26:54.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.263852 2301 connection.cc:647] client connection to 127.1.193.66:39259 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:26:54.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.264220 2301 consensus_peers.cc:457] T 02d12d6ae86c466fb511f0ddef2f6fc7 P b9bbb727f40b43e2956d6f77e8505f0b -> Peer 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259): Couldn't send request to peer 134794f36230463ea5d0f3f800c08596. Status: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104). This is attempt 1: this message will repeat every 5th retry. 01:26:54.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.264632 2301 consensus_peers.cc:457] T c0626d9f34c84a0488641522977ae3d0 P b9bbb727f40b43e2956d6f77e8505f0b -> Peer 134794f36230463ea5d0f3f800c08596 (127.1.193.66:39259): Couldn't send request to peer 134794f36230463ea5d0f3f800c08596. Status: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104). This is attempt 1: this message will repeat every 5th retry. 01:26:54.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.265434 2301 connection.cc:320] Error closing socket: Network error: TlsSocket::Close: Broken pipe 01:26:54.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.266083 2303 connection.cc:647] server connection from 127.1.193.66:45482 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.66:45482): Connection reset by peer (error 104) 01:26:54.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:54.266221 1797 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 2287 01:26:54.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.320690 1886 connection.cc:647] server connection from 127.1.193.67:33037 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.67:33037): Connection reset by peer (error 104) 01:26:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.321691 1951 connection.cc:647] server connection from 127.1.193.67:33427 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.67:33427): Connection reset by peer (error 104) 01:26:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:26:54.321702 1815 connection.cc:647] server connection from 127.1.193.67:44706 recv error: Network error: failed to read from TLS socket (remote: 127.1.193.67:44706): Connection reset by peer (error 104) 01:26:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:54.322227 1797 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 1806 01:26:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:54.354204 1797 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 1868 01:26:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:26:54.387015 1797 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 1936 ]]> org.junit.runners.model.TestTimedOutException: test timed out after 60000 milliseconds at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.stumbleupon.async.Deferred.doJoin(Deferred.java:1136) at com.stumbleupon.async.Deferred.join(Deferred.java:1019) at org.apache.kudu.client.KuduSession.apply(KuduSession.java:81) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:163) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:150) at org.apache.kudu.client.ITClientStress$1.run(ITClientStress.java:80) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:2022) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at org.apache.kudu.client.ITClientStress.runTasks(ITClientStress.java:86) at org.apache.kudu.client.ITClientStress.testMultipleSessions(ITClientStress.java:147) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) /127.5.89.125:60269] using SASL/PLAIN 01:27:02.065 [DEBUG - New I/O worker #2145] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.5.89.126 01:27:02.065 [DEBUG - New I/O worker #2144] (Negotiator.java:764) Authenticated connection [id: 0x020db9af, /127.5.89.124:55093 => /127.5.89.124:42353] using SASL/PLAIN 01:27:02.069 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:27:02.069 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=18), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.89.125:60269, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.89.124:42353, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.89.126:45573, [17ms] Sub RPC ConnectToMaster: received response from server master-127.5.89.125:60269: OK, [17ms] Sub RPC ConnectToMaster: received response from server master-127.5.89.124:42353: OK, deferred=null) after lookup 01:27:02.070 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1644571674 returned Deferred@1715057794(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after retry RPC@1644571674), errback=(continuation of Deferred@591078945 after retry RPC@1644571674)), so the following Deferred is getting paused: Deferred@591078945(state=PAUSED, result=Deferred@1715057794, callback=org.apache.kudu.client.AsyncKuduClient$1@416743fb -> org.apache.kudu.client.AsyncKuduClient$2@12d2c131 -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 01:27:02.072 [DEBUG - New I/O worker #2145] (Negotiator.java:764) Authenticated connection [id: 0x4aa939dc, /127.5.89.126:50720 => /127.5.89.126:45573] using SASL/PLAIN 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.071081 5581 catalog_manager.cc:1386] Servicing CreateTable request from {username='jenkins'} at 127.5.89.125:55603: 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "testMultipleSessions" 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) schema { 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: true 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column1_i" 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:27:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column2_i" 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column3_s" 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: STRING 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: true 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) encoding: DICT_ENCODING 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) compression: LZ4 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 4096 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column4_b" 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: BOOL 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) partition_schema { 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) range_schema { 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:27:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:02.095031 5581 catalog_manager.cc:1537] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table testMultipleSessions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 01:27:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.131572 5972 heartbeater.cc:467] Master 127.5.89.125:60269 was elected leader, sending a full tablet report... 01:27:02.146 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@416743fb@1097286651 returned Deferred@962918989(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@4314d96e -> (continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$1@416743fb@1097286651), errback=passthrough -> (continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$1@416743fb@1097286651)), so the following Deferred is getting paused: Deferred@591078945(state=PAUSED, result=Deferred@962918989, callback=org.apache.kudu.client.AsyncKuduClient$2@12d2c131 -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:27:02.149 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:761) Opened table a35561ccf34e42188d22471fe00c36be 01:27:02.149 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905)), so the following Deferred is getting paused: Deferred@591078945(state=PAUSED, result=Deferred@949432940, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 01:27:02.154 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014)), so the following Deferred is getting paused: Deferred@576754752(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.175 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659)), so the following Deferred is getting paused: Deferred@1180674078(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.186167 5770 tablet_service.cc:870] Processing CreateTablet for tablet 352ac1600b824dd8bec48ce92773d445 (table=testMultipleSessions [id=a35561ccf34e42188d22471fe00c36be]), partition=RANGE (key) PARTITION UNBOUNDED 01:27:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.187294 5770 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 352ac1600b824dd8bec48ce92773d445. 1 dirs total, 0 dirs full, 0 dirs failed 01:27:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.186980 6046 tablet_service.cc:870] Processing CreateTablet for tablet 352ac1600b824dd8bec48ce92773d445 (table=testMultipleSessions [id=a35561ccf34e42188d22471fe00c36be]), partition=RANGE (key) PARTITION UNBOUNDED 01:27:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.188169 6046 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 352ac1600b824dd8bec48ce92773d445. 1 dirs total, 0 dirs full, 0 dirs failed 01:27:02.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.189985 5905 tablet_service.cc:870] Processing CreateTablet for tablet 352ac1600b824dd8bec48ce92773d445 (table=testMultipleSessions [id=a35561ccf34e42188d22471fe00c36be]), partition=RANGE (key) PARTITION UNBOUNDED 01:27:02.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.191032 5905 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 352ac1600b824dd8bec48ce92773d445. 1 dirs total, 0 dirs full, 0 dirs failed 01:27:02.194 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964)), so the following Deferred is getting paused: Deferred@431097450(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.205566 6046 ts_tablet_manager.cc:1175] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Registered tablet (data state: TABLET_DATA_READY) 01:27:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.208175 6151 ts_tablet_manager.cc:1030] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Bootstrapping tablet 01:27:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.209589 6151 tablet_bootstrap.cc:438] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Bootstrap starting. 01:27:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.213822 6151 tablet_bootstrap.cc:592] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: No blocks or log segments found. Creating new log. 01:27:02.214 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345)), so the following Deferred is getting paused: Deferred@854233430(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.216061 6151 log.cc:525] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Log is configured to *not* fsync() on all Append() calls 01:27:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.217321 5905 ts_tablet_manager.cc:1175] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Registered tablet (data state: TABLET_DATA_READY) 01:27:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.219358 5770 ts_tablet_manager.cc:1175] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Registered tablet (data state: TABLET_DATA_READY) 01:27:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.219862 6153 ts_tablet_manager.cc:1030] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Bootstrapping tablet 01:27:02.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.221285 6153 tablet_bootstrap.cc:438] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Bootstrap starting. 01:27:02.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.222693 6154 ts_tablet_manager.cc:1030] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Bootstrapping tablet 01:27:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.223908 6154 tablet_bootstrap.cc:438] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Bootstrap starting. 01:27:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.224500 6151 tablet_bootstrap.cc:438] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: No bootstrap required, opened a new log 01:27:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.225116 6151 ts_tablet_manager.cc:1047] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Time spent bootstrapping tablet: real 0.016s user 0.007s sys 0.003s 01:27:02.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.227286 6154 tablet_bootstrap.cc:592] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: No blocks or log segments found. Creating new log. 01:27:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.229066 6154 log.cc:525] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Log is configured to *not* fsync() on all Append() calls 01:27:02.234 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368)), so the following Deferred is getting paused: Deferred@906218613(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.238394 6153 tablet_bootstrap.cc:592] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: No blocks or log segments found. Creating new log. 01:27:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.240646 6154 tablet_bootstrap.cc:438] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: No bootstrap required, opened a new log 01:27:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.241504 6153 log.cc:525] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Log is configured to *not* fsync() on all Append() calls 01:27:02.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.241577 6154 ts_tablet_manager.cc:1047] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Time spent bootstrapping tablet: real 0.018s user 0.006s sys 0.008s 01:27:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.251749 6151 raft_consensus.cc:340] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.253330 6151 raft_consensus.cc:366] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:27:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.253895 6151 raft_consensus.cc:705] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01faeb22d9e448ac83be7ed9e2e37b64, State: Initialized, Role: FOLLOWER 01:27:02.254 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155)), so the following Deferred is getting paused: Deferred@797829701(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.255291 6151 consensus_queue.cc:229] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.259217 6153 tablet_bootstrap.cc:438] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: No bootstrap required, opened a new log 01:27:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.259778 6153 ts_tablet_manager.cc:1047] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Time spent bootstrapping tablet: real 0.039s user 0.008s sys 0.006s 01:27:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.266595 6110 heartbeater.cc:467] Master 127.5.89.125:60269 was elected leader, sending a full tablet report... 01:27:02.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.272693 6151 ts_tablet_manager.cc:1075] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Time spent starting tablet: real 0.047s user 0.027s sys 0.010s 01:27:02.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.281980 6153 raft_consensus.cc:340] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.282682 6153 raft_consensus.cc:366] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:27:02.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.283080 6153 raft_consensus.cc:705] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42c3f16052ca4d0dbd066f5c63031d10, State: Initialized, Role: FOLLOWER 01:27:02.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.283846 6153 consensus_queue.cc:229] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.284857 6154 raft_consensus.cc:340] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.286020 6154 raft_consensus.cc:366] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:27:02.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.286401 6154 raft_consensus.cc:705] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 14bbf4194d1c467b9d518d02d40bb86e, State: Initialized, Role: FOLLOWER 01:27:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.287927 6153 ts_tablet_manager.cc:1075] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Time spent starting tablet: real 0.028s user 0.022s sys 0.003s 01:27:02.295 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155) -> (continuation of Deferred@1615646913 after org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155) -> (continuation of Deferred@1615646913 after org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072)), so the following Deferred is getting paused: Deferred@1615646913(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.296871 6154 consensus_queue.cc:229] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.300591 6159 raft_consensus.cc:472] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:27:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.301206 6159 raft_consensus.cc:494] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.319350 6154 ts_tablet_manager.cc:1075] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Time spent starting tablet: real 0.077s user 0.025s sys 0.008s 01:27:02.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.321279 6159 leader_election.cc:296] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 01faeb22d9e448ac83be7ed9e2e37b64 (127.5.89.67:40645), 42c3f16052ca4d0dbd066f5c63031d10 (127.5.89.66:38592) 01:27:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:02.328872 5978 tablet.cc:1783] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:27:02.336 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@7443ede1@1950608865 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155) -> (continuation of Deferred@1615646913 after org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072) -> (continuation of Deferred@878505710 after org.apache.kudu.client.AsyncKuduClient$15@7443ede1@1950608865), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155) -> (continuation of Deferred@1615646913 after org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072) -> (continuation of Deferred@878505710 after org.apache.kudu.client.AsyncKuduClient$15@7443ede1@1950608865)), so the following Deferred is getting paused: Deferred@878505710(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.347399 6066 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "352ac1600b824dd8bec48ce92773d445" candidate_uuid: "14bbf4194d1c467b9d518d02d40bb86e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" is_pre_election: true 01:27:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.348265 6066 raft_consensus.cc:2346] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 14bbf4194d1c467b9d518d02d40bb86e in term 0. 01:27:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.349931 5712 leader_election.cc:310] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01faeb22d9e448ac83be7ed9e2e37b64, 14bbf4194d1c467b9d518d02d40bb86e; no voters: 01:27:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.349812 5925 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "352ac1600b824dd8bec48ce92773d445" candidate_uuid: "14bbf4194d1c467b9d518d02d40bb86e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42c3f16052ca4d0dbd066f5c63031d10" is_pre_election: true 01:27:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.350458 5925 raft_consensus.cc:2346] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 14bbf4194d1c467b9d518d02d40bb86e in term 0. 01:27:02.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.350973 6159 raft_consensus.cc:2640] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Leader pre-election won for term 1 01:27:02.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.351227 6159 raft_consensus.cc:472] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:27:02.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.351574 6159 raft_consensus.cc:2886] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 0 FOLLOWER]: Advancing to term 1 01:27:02.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.355763 6159 raft_consensus.cc:494] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.357347 6159 leader_election.cc:296] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [CANDIDATE]: Term 1 election: Requested vote from peers 01faeb22d9e448ac83be7ed9e2e37b64 (127.5.89.67:40645), 42c3f16052ca4d0dbd066f5c63031d10 (127.5.89.66:38592) 01:27:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.358609 5925 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "352ac1600b824dd8bec48ce92773d445" candidate_uuid: "14bbf4194d1c467b9d518d02d40bb86e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42c3f16052ca4d0dbd066f5c63031d10" 01:27:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.358829 6066 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "352ac1600b824dd8bec48ce92773d445" candidate_uuid: "14bbf4194d1c467b9d518d02d40bb86e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" 01:27:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.359244 5925 raft_consensus.cc:2886] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 0 FOLLOWER]: Advancing to term 1 01:27:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.359417 6066 raft_consensus.cc:2886] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 0 FOLLOWER]: Advancing to term 1 01:27:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.365713 5925 raft_consensus.cc:2346] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 14bbf4194d1c467b9d518d02d40bb86e in term 1. 01:27:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.367194 6066 raft_consensus.cc:2346] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 14bbf4194d1c467b9d518d02d40bb86e in term 1. 01:27:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.367223 5702 leader_election.cc:310] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 14bbf4194d1c467b9d518d02d40bb86e, 42c3f16052ca4d0dbd066f5c63031d10; no voters: 01:27:02.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.368154 6159 raft_consensus.cc:2640] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 1 FOLLOWER]: Leader election won for term 1 01:27:02.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.369210 6159 raft_consensus.cc:667] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [term 1 LEADER]: Becoming Leader. State: Replica: 14bbf4194d1c467b9d518d02d40bb86e, State: Running, Role: LEADER 01:27:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.370030 6159 consensus_queue.cc:206] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } } 01:27:02.374 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@5a585f78@1515741048 returned Deferred@949432940(state=PENDING, result=null, callback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155) -> (continuation of Deferred@1615646913 after org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072) -> (continuation of Deferred@878505710 after org.apache.kudu.client.AsyncKuduClient$15@7443ede1@1950608865) -> (continuation of Deferred@1544832554 after org.apache.kudu.client.AsyncKuduClient$15@5a585f78@1515741048), errback=(continuation of Deferred@591078945 after org.apache.kudu.client.AsyncKuduClient$2@12d2c131@315801905) -> (continuation of Deferred@576754752 after org.apache.kudu.client.AsyncKuduClient$15@3cc8b706@1019787014) -> (continuation of Deferred@1180674078 after org.apache.kudu.client.AsyncKuduClient$15@475ae6cb@1197139659) -> (continuation of Deferred@431097450 after org.apache.kudu.client.AsyncKuduClient$15@3f564754@1062618964) -> (continuation of Deferred@854233430 after org.apache.kudu.client.AsyncKuduClient$15@9cec729@164546345) -> (continuation of Deferred@906218613 after org.apache.kudu.client.AsyncKuduClient$15@3f8b0cb8@1066077368) -> (continuation of Deferred@797829701 after org.apache.kudu.client.AsyncKuduClient$15@490ba213@1225499155) -> (continuation of Deferred@1615646913 after org.apache.kudu.client.AsyncKuduClient$15@4884ddc0@1216667072) -> (continuation of Deferred@878505710 after org.apache.kudu.client.AsyncKuduClient$15@7443ede1@1950608865) -> (continuation of Deferred@1544832554 after org.apache.kudu.client.AsyncKuduClient$15@5a585f78@1515741048)), so the following Deferred is getting paused: Deferred@1544832554(state=PAUSED, result=Deferred@949432940, callback=, errback=) 01:27:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.378322 5580 catalog_manager.cc:3791] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e reported cstate change: term changed from 0 to 1, leader changed from to 14bbf4194d1c467b9d518d02d40bb86e (127.5.89.65). New cstate: current_term: 1 leader_uuid: "14bbf4194d1c467b9d518d02d40bb86e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14bbf4194d1c467b9d518d02d40bb86e" member_type: VOTER last_known_addr { host: "127.5.89.65" port: 34277 } health_report { overall_health: HEALTHY } } } 01:27:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:02.408685 5845 tablet.cc:1783] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:27:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:02.437970 6116 tablet.cc:1783] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:27:02.734 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.735 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.735 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=11), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [11ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.746 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1680549670 returned Deferred@2081425220(state=PENDING, result=null, callback=(continuation of Deferred@38484986 after retry RPC@1680549670), errback=(continuation of Deferred@38484986 after retry RPC@1680549670)), so the following Deferred is getting paused: Deferred@38484986(state=PAUSED, result=Deferred@2081425220, callback=org.apache.kudu.client.AsyncKuduSession$2@2996b9eb -> passthrough -> wakeup thread pool-480-thread-9, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1439a9e8 -> wakeup thread pool-480-thread-9) 01:27:02.747 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.747 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.748 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=23), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [23ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.748 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1760536076 returned Deferred@857785344(state=PENDING, result=null, callback=(continuation of Deferred@795175534 after retry RPC@1760536076), errback=(continuation of Deferred@795175534 after retry RPC@1760536076)), so the following Deferred is getting paused: Deferred@795175534(state=PAUSED, result=Deferred@857785344, callback=org.apache.kudu.client.AsyncKuduSession$2@76cae083 -> passthrough -> wakeup thread pool-480-thread-20, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6380f790 -> wakeup thread pool-480-thread-20) 01:27:02.748 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.749 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.749 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=25), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [24ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.749 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1446326052 returned Deferred@1384581526(state=PENDING, result=null, callback=(continuation of Deferred@1936205245 after retry RPC@1446326052), errback=(continuation of Deferred@1936205245 after retry RPC@1446326052)), so the following Deferred is getting paused: Deferred@1936205245(state=PAUSED, result=Deferred@1384581526, callback=org.apache.kudu.client.AsyncKuduSession$2@17b07037 -> passthrough -> wakeup thread pool-480-thread-22, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680331ea -> wakeup thread pool-480-thread-22) 01:27:02.750 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.750 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.750 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=25), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [25ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.751 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@35806917 returned Deferred@819320561(state=PENDING, result=null, callback=(continuation of Deferred@611334420 after retry RPC@35806917), errback=(continuation of Deferred@611334420 after retry RPC@35806917)), so the following Deferred is getting paused: Deferred@611334420(state=PAUSED, result=Deferred@819320561, callback=org.apache.kudu.client.AsyncKuduSession$2@550dd011 -> passthrough -> wakeup thread pool-480-thread-4, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45c96803 -> wakeup thread pool-480-thread-4) 01:27:02.751 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.751 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.752 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=28), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [28ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.752 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1538608149 returned Deferred@2124969710(state=PENDING, result=null, callback=(continuation of Deferred@1168026738 after retry RPC@1538608149), errback=(continuation of Deferred@1168026738 after retry RPC@1538608149)), so the following Deferred is getting paused: Deferred@1168026738(state=PAUSED, result=Deferred@2124969710, callback=org.apache.kudu.client.AsyncKuduSession$2@4b994db2 -> passthrough -> wakeup thread pool-480-thread-8, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a3d7530 -> wakeup thread pool-480-thread-8) 01:27:02.755 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.756 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.756 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=33), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [32ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.757 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@459091526 returned Deferred@964461605(state=PENDING, result=null, callback=(continuation of Deferred@863810576 after retry RPC@459091526), errback=(continuation of Deferred@863810576 after retry RPC@459091526)), so the following Deferred is getting paused: Deferred@863810576(state=PAUSED, result=Deferred@964461605, callback=org.apache.kudu.client.AsyncKuduSession$2@437b978b -> passthrough -> wakeup thread pool-480-thread-16, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3126bebc -> wakeup thread pool-480-thread-16) 01:27:02.758 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.759 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.760 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=35), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [34ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.761 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1822841105 returned Deferred@976010401(state=PENDING, result=null, callback=(continuation of Deferred@636793760 after retry RPC@1822841105), errback=(continuation of Deferred@636793760 after retry RPC@1822841105)), so the following Deferred is getting paused: Deferred@636793760(state=PAUSED, result=Deferred@976010401, callback=org.apache.kudu.client.AsyncKuduSession$2@21a33afc -> passthrough -> wakeup thread pool-480-thread-24, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f939108 -> wakeup thread pool-480-thread-24) 01:27:02.762 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.763 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.763 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=38), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [38ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.768 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@537261039 returned Deferred@552353276(state=PENDING, result=null, callback=(continuation of Deferred@404789191 after retry RPC@537261039), errback=(continuation of Deferred@404789191 after retry RPC@537261039)), so the following Deferred is getting paused: Deferred@404789191(state=PAUSED, result=Deferred@552353276, callback=org.apache.kudu.client.AsyncKuduSession$2@7b584e1a -> passthrough -> wakeup thread pool-480-thread-6, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@744d4015 -> wakeup thread pool-480-thread-6) 01:27:02.769 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.769 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.770 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=45), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [45ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.771 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@890212541 returned Deferred@688760692(state=PENDING, result=null, callback=(continuation of Deferred@270271697 after retry RPC@890212541), errback=(continuation of Deferred@270271697 after retry RPC@890212541)), so the following Deferred is getting paused: Deferred@270271697(state=PAUSED, result=Deferred@688760692, callback=org.apache.kudu.client.AsyncKuduSession$2@e4fb32f -> passthrough -> wakeup thread pool-480-thread-21, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@478c5023 -> wakeup thread pool-480-thread-21) 01:27:02.771 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.772 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.773 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=48), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [47ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.774 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@58620321 returned Deferred@1932840851(state=PENDING, result=null, callback=(continuation of Deferred@1170449 after retry RPC@58620321), errback=(continuation of Deferred@1170449 after retry RPC@58620321)), so the following Deferred is getting paused: Deferred@1170449(state=PAUSED, result=Deferred@1932840851, callback=org.apache.kudu.client.AsyncKuduSession$2@25b98978 -> passthrough -> wakeup thread pool-480-thread-7, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55fa4f1f -> wakeup thread pool-480-thread-7) 01:27:02.774 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.774 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.790 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=66), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [51ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.790 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@6769252 returned Deferred@244903725(state=PENDING, result=null, callback=(continuation of Deferred@1736660431 after retry RPC@6769252), errback=(continuation of Deferred@1736660431 after retry RPC@6769252)), so the following Deferred is getting paused: Deferred@1736660431(state=PAUSED, result=Deferred@244903725, callback=org.apache.kudu.client.AsyncKuduSession$2@60067c97 -> passthrough -> wakeup thread pool-480-thread-1, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a083e99 -> wakeup thread pool-480-thread-1) 01:27:02.791 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.791 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.794289 5925 raft_consensus.cc:1181] T 352ac1600b824dd8bec48ce92773d445 P 42c3f16052ca4d0dbd066f5c63031d10 [term 1 FOLLOWER]: Refusing update from remote peer 14bbf4194d1c467b9d518d02d40bb86e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:27:02.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.796934 6159 consensus_queue.cc:984] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [LEADER]: Connected to new peer: Peer: permanent_uuid: "42c3f16052ca4d0dbd066f5c63031d10" member_type: VOTER last_known_addr { host: "127.5.89.66" port: 38592 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:27:02.792 [DEBUG - New I/O worker #2146] (Negotiator.java:764) Authenticated connection [id: 0x433fdb5f, /127.5.89.65:53105 => /127.5.89.65:34277] using TOKEN/null 01:27:02.797 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=72), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [66ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.803 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@320881522 returned Deferred@1861239284(state=PENDING, result=null, callback=(continuation of Deferred@1036089977 after retry RPC@320881522), errback=(continuation of Deferred@1036089977 after retry RPC@320881522)), so the following Deferred is getting paused: Deferred@1036089977(state=PAUSED, result=Deferred@1861239284, callback=org.apache.kudu.client.AsyncKuduSession$2@26a677ae -> passthrough -> wakeup thread pool-480-thread-15, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@378c8bdb -> wakeup thread pool-480-thread-15) 01:27:02.811 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.812 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.813 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=88), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [79ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.814 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1528791871 returned Deferred@1832597655(state=PENDING, result=null, callback=(continuation of Deferred@2025142681 after retry RPC@1528791871), errback=(continuation of Deferred@2025142681 after retry RPC@1528791871)), so the following Deferred is getting paused: Deferred@2025142681(state=PAUSED, result=Deferred@1832597655, callback=org.apache.kudu.client.AsyncKuduSession$2@2bb4b707 -> passthrough -> wakeup thread pool-480-thread-26, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@c95df35 -> wakeup thread pool-480-thread-26) 01:27:02.815 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.815 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.815 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=90), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [90ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.816 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@945729968 returned Deferred@1777741523(state=PENDING, result=null, callback=(continuation of Deferred@1744085093 after retry RPC@945729968), errback=(continuation of Deferred@1744085093 after retry RPC@945729968)), so the following Deferred is getting paused: Deferred@1744085093(state=PAUSED, result=Deferred@1777741523, callback=org.apache.kudu.client.AsyncKuduSession$2@45bab7d9 -> passthrough -> wakeup thread pool-480-thread-10, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8245def -> wakeup thread pool-480-thread-10) 01:27:02.821 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.821 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.823 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=97), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [96ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.823 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1350331022 returned Deferred@306777964(state=PENDING, result=null, callback=(continuation of Deferred@48068227 after retry RPC@1350331022), errback=(continuation of Deferred@48068227 after retry RPC@1350331022)), so the following Deferred is getting paused: Deferred@48068227(state=PAUSED, result=Deferred@306777964, callback=org.apache.kudu.client.AsyncKuduSession$2@283c8812 -> passthrough -> wakeup thread pool-480-thread-3, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1b27af81 -> wakeup thread pool-480-thread-3) 01:27:02.824 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.824 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.827 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=103), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [99ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.828 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1095754214 returned Deferred@676331042(state=PENDING, result=null, callback=(continuation of Deferred@1686081313 after retry RPC@1095754214), errback=(continuation of Deferred@1686081313 after retry RPC@1095754214)), so the following Deferred is getting paused: Deferred@1686081313(state=PAUSED, result=Deferred@676331042, callback=org.apache.kudu.client.AsyncKuduSession$2@42826f88 -> passthrough -> wakeup thread pool-480-thread-18, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7c7cf6ef -> wakeup thread pool-480-thread-18) 01:27:02.828 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.829 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.830 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=104), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [103ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.833 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@722844076 returned Deferred@856374632(state=PENDING, result=null, callback=(continuation of Deferred@1654975150 after retry RPC@722844076), errback=(continuation of Deferred@1654975150 after retry RPC@722844076)), so the following Deferred is getting paused: Deferred@1654975150(state=PAUSED, result=Deferred@856374632, callback=org.apache.kudu.client.AsyncKuduSession$2@7974a39b -> passthrough -> wakeup thread pool-480-thread-14, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f2aadc0 -> wakeup thread pool-480-thread-14) 01:27:02.840 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.842 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.843 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=118), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [115ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.845 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@362819376 returned Deferred@1343510192(state=PENDING, result=null, callback=(continuation of Deferred@211836754 after retry RPC@362819376), errback=(continuation of Deferred@211836754 after retry RPC@362819376)), so the following Deferred is getting paused: Deferred@211836754(state=PAUSED, result=Deferred@1343510192, callback=org.apache.kudu.client.AsyncKuduSession$2@264eb7ab -> passthrough -> wakeup thread pool-480-thread-25, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3203d5fd -> wakeup thread pool-480-thread-25) 01:27:02.845 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.846 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.846 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=120), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [120ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.848 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1087415515 returned Deferred@1317288284(state=PENDING, result=null, callback=(continuation of Deferred@497298632 after retry RPC@1087415515), errback=(continuation of Deferred@497298632 after retry RPC@1087415515)), so the following Deferred is getting paused: Deferred@497298632(state=PAUSED, result=Deferred@1317288284, callback=org.apache.kudu.client.AsyncKuduSession$2@3124aebc -> passthrough -> wakeup thread pool-480-thread-5, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7417323f -> wakeup thread pool-480-thread-5) 01:27:02.850 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.850 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.850 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=125), Traces: [0ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [125ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.851 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@456567608 returned Deferred@659031802(state=PENDING, result=null, callback=(continuation of Deferred@864882145 after retry RPC@456567608), errback=(continuation of Deferred@864882145 after retry RPC@456567608)), so the following Deferred is getting paused: Deferred@864882145(state=PAUSED, result=Deferred@659031802, callback=org.apache.kudu.client.AsyncKuduSession$2@7ec64765 -> passthrough -> wakeup thread pool-480-thread-28, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@50075cee -> wakeup thread pool-480-thread-28) 01:27:02.852 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.852 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.853 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=123), Traces: [0ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [123ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.853 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@933232578 returned Deferred@1814433493(state=PENDING, result=null, callback=(continuation of Deferred@538411355 after retry RPC@933232578), errback=(continuation of Deferred@538411355 after retry RPC@933232578)), so the following Deferred is getting paused: Deferred@538411355(state=PAUSED, result=Deferred@1814433493, callback=org.apache.kudu.client.AsyncKuduSession$2@4a9d499b -> passthrough -> wakeup thread pool-480-thread-39, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b4cee5 -> wakeup thread pool-480-thread-39) 01:27:02.853 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.854 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.854 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=124), Traces: [0ms] refreshing cache from master, [2ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [123ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.855 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1580742848 returned Deferred@1119446789(state=PENDING, result=null, callback=(continuation of Deferred@1657286488 after retry RPC@1580742848), errback=(continuation of Deferred@1657286488 after retry RPC@1580742848)), so the following Deferred is getting paused: Deferred@1657286488(state=PAUSED, result=Deferred@1119446789, callback=org.apache.kudu.client.AsyncKuduSession$2@46af1602 -> passthrough -> wakeup thread pool-480-thread-40, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@594ed82f -> wakeup thread pool-480-thread-40) 01:27:02.855 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.857 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.865 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=135), Traces: [0ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [126ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.865 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1346772300 returned Deferred@1935104755(state=PENDING, result=null, callback=(continuation of Deferred@1299598749 after retry RPC@1346772300), errback=(continuation of Deferred@1299598749 after retry RPC@1346772300)), so the following Deferred is getting paused: Deferred@1299598749(state=PAUSED, result=Deferred@1935104755, callback=org.apache.kudu.client.AsyncKuduSession$2@62d66bbc -> passthrough -> wakeup thread pool-480-thread-38, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29ab6d7 -> wakeup thread pool-480-thread-38) 01:27:02.866 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.866 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.867 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=137), Traces: [0ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [137ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.869 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1272177845 returned Deferred@875975415(state=PENDING, result=null, callback=(continuation of Deferred@1701594182 after retry RPC@1272177845), errback=(continuation of Deferred@1701594182 after retry RPC@1272177845)), so the following Deferred is getting paused: Deferred@1701594182(state=PAUSED, result=Deferred@875975415, callback=org.apache.kudu.client.AsyncKuduSession$2@32176a32 -> passthrough -> wakeup thread pool-480-thread-37, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6635d689 -> wakeup thread pool-480-thread-37) 01:27:02.870 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.872 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.873 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=144), Traces: [0ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [142ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.874 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1620214098 returned Deferred@1742448498(state=PENDING, result=null, callback=(continuation of Deferred@1846806337 after retry RPC@1620214098), errback=(continuation of Deferred@1846806337 after retry RPC@1620214098)), so the following Deferred is getting paused: Deferred@1846806337(state=PAUSED, result=Deferred@1742448498, callback=org.apache.kudu.client.AsyncKuduSession$2@26dd1594 -> passthrough -> wakeup thread pool-480-thread-36, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5231980c -> wakeup thread pool-480-thread-36) 01:27:02.875 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.875 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.876 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=147), Traces: [0ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [147ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.877 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1760249103 returned Deferred@638282076(state=PENDING, result=null, callback=(continuation of Deferred@1756674492 after retry RPC@1760249103), errback=(continuation of Deferred@1756674492 after retry RPC@1760249103)), so the following Deferred is getting paused: Deferred@1756674492(state=PAUSED, result=Deferred@638282076, callback=org.apache.kudu.client.AsyncKuduSession$2@369bbd5e -> passthrough -> wakeup thread pool-480-thread-35, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46a55bf -> wakeup thread pool-480-thread-35) 01:27:02.879 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.882 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.883 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=154), Traces: [0ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [151ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.887 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@2044104167 returned Deferred@878879288(state=PENDING, result=null, callback=(continuation of Deferred@1601734823 after retry RPC@2044104167), errback=(continuation of Deferred@1601734823 after retry RPC@2044104167)), so the following Deferred is getting paused: Deferred@1601734823(state=PAUSED, result=Deferred@878879288, callback=org.apache.kudu.client.AsyncKuduSession$2@969f95a -> passthrough -> wakeup thread pool-480-thread-34, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@441df425 -> wakeup thread pool-480-thread-34) 01:27:02.887 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.893 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.894 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=166), Traces: [0ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [160ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.897 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1708129788 returned Deferred@2129230930(state=PENDING, result=null, callback=(continuation of Deferred@940374524 after retry RPC@1708129788), errback=(continuation of Deferred@940374524 after retry RPC@1708129788)), so the following Deferred is getting paused: Deferred@940374524(state=PAUSED, result=Deferred@2129230930, callback=org.apache.kudu.client.AsyncKuduSession$2@139b9e04 -> passthrough -> wakeup thread pool-480-thread-33, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@9a1a196 -> wakeup thread pool-480-thread-33) 01:27:02.913 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.914 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.914 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=187), Traces: [0ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [186ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.905026 6066 raft_consensus.cc:1181] T 352ac1600b824dd8bec48ce92773d445 P 01faeb22d9e448ac83be7ed9e2e37b64 [term 1 FOLLOWER]: Refusing update from remote peer 14bbf4194d1c467b9d518d02d40bb86e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 1. (index mismatch) 01:27:02.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:02.908746 6159 consensus_queue.cc:984] T 352ac1600b824dd8bec48ce92773d445 P 14bbf4194d1c467b9d518d02d40bb86e [LEADER]: Connected to new peer: Peer: permanent_uuid: "01faeb22d9e448ac83be7ed9e2e37b64" member_type: VOTER last_known_addr { host: "127.5.89.67" port: 40645 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:27:02.915 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1017071388 returned Deferred@1661180522(state=PENDING, result=null, callback=(continuation of Deferred@749159344 after retry RPC@1017071388), errback=(continuation of Deferred@749159344 after retry RPC@1017071388)), so the following Deferred is getting paused: Deferred@749159344(state=PAUSED, result=Deferred@1661180522, callback=org.apache.kudu.client.AsyncKuduSession$2@280eeaa3 -> passthrough -> wakeup thread pool-480-thread-32, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6e6fff19 -> wakeup thread pool-480-thread-32) 01:27:02.916 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.916 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.924 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=197), Traces: [0ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [188ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.929 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@297855092 returned Deferred@132944703(state=PENDING, result=null, callback=(continuation of Deferred@1483221651 after retry RPC@297855092), errback=(continuation of Deferred@1483221651 after retry RPC@297855092)), so the following Deferred is getting paused: Deferred@1483221651(state=PAUSED, result=Deferred@132944703, callback=org.apache.kudu.client.AsyncKuduSession$2@49ee81b0 -> passthrough -> wakeup thread pool-480-thread-31, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5f7b43b1 -> wakeup thread pool-480-thread-31) 01:27:02.929 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.930 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.930 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=203), Traces: [0ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [203ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.931 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1022187018 returned Deferred@633917366(state=PENDING, result=null, callback=(continuation of Deferred@404109758 after retry RPC@1022187018), errback=(continuation of Deferred@404109758 after retry RPC@1022187018)), so the following Deferred is getting paused: Deferred@404109758(state=PAUSED, result=Deferred@633917366, callback=org.apache.kudu.client.AsyncKuduSession$2@1df587c6 -> passthrough -> wakeup thread pool-480-thread-30, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b7edf74 -> wakeup thread pool-480-thread-30) 01:27:02.932 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.932 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.933 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=206), Traces: [0ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [206ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.934 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@673713362 returned Deferred@31071136(state=PENDING, result=null, callback=(continuation of Deferred@815818730 after retry RPC@673713362), errback=(continuation of Deferred@815818730 after retry RPC@673713362)), so the following Deferred is getting paused: Deferred@815818730(state=PAUSED, result=Deferred@31071136, callback=org.apache.kudu.client.AsyncKuduSession$2@d188bf5 -> passthrough -> wakeup thread pool-480-thread-29, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@653e4818 -> wakeup thread pool-480-thread-29) 01:27:02.936 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.938 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.939 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=213), Traces: [0ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [211ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.939 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@887638020 returned Deferred@130344516(state=PENDING, result=null, callback=(continuation of Deferred@1321898602 after retry RPC@887638020), errback=(continuation of Deferred@1321898602 after retry RPC@887638020)), so the following Deferred is getting paused: Deferred@1321898602(state=PAUSED, result=Deferred@130344516, callback=org.apache.kudu.client.AsyncKuduSession$2@59a37d23 -> passthrough -> wakeup thread pool-480-thread-17, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f7da67b -> wakeup thread pool-480-thread-17) 01:27:02.940 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.940 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.944 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=218), Traces: [0ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [214ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.945 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@254509846 returned Deferred@367532379(state=PENDING, result=null, callback=(continuation of Deferred@1465918624 after retry RPC@254509846), errback=(continuation of Deferred@1465918624 after retry RPC@254509846)), so the following Deferred is getting paused: Deferred@1465918624(state=PAUSED, result=Deferred@367532379, callback=org.apache.kudu.client.AsyncKuduSession$2@156c1e49 -> passthrough -> wakeup thread pool-480-thread-13, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@460c1cc1 -> wakeup thread pool-480-thread-13) 01:27:02.956 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.956 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.957 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=231), Traces: [0ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [222ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.958 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@335280026 returned Deferred@1172866145(state=PENDING, result=null, callback=(continuation of Deferred@805524675 after retry RPC@335280026), errback=(continuation of Deferred@805524675 after retry RPC@335280026)), so the following Deferred is getting paused: Deferred@805524675(state=PAUSED, result=Deferred@1172866145, callback=org.apache.kudu.client.AsyncKuduSession$2@16cfbc16 -> passthrough -> wakeup thread pool-480-thread-19, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ac4ce72 -> wakeup thread pool-480-thread-19) 01:27:02.958 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.967 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.967 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=242), Traces: [0ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [233ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.969 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1536070343 returned Deferred@1904308281(state=PENDING, result=null, callback=(continuation of Deferred@440749331 after retry RPC@1536070343), errback=(continuation of Deferred@440749331 after retry RPC@1536070343)), so the following Deferred is getting paused: Deferred@440749331(state=PAUSED, result=Deferred@1904308281, callback=org.apache.kudu.client.AsyncKuduSession$2@660ea777 -> passthrough -> wakeup thread pool-480-thread-12, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@493909db -> wakeup thread pool-480-thread-12) 01:27:02.969 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.972 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.973 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=239), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [236ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.973 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@868460669 returned Deferred@64902851(state=PENDING, result=null, callback=(continuation of Deferred@276331728 after retry RPC@868460669), errback=(continuation of Deferred@276331728 after retry RPC@868460669)), so the following Deferred is getting paused: Deferred@276331728(state=PAUSED, result=Deferred@64902851, callback=org.apache.kudu.client.AsyncKuduSession$2@110ec8d5 -> passthrough -> wakeup thread pool-480-thread-42, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3fa01ac9 -> wakeup thread pool-480-thread-42) 01:27:02.974 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.976 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.984 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=250), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [241ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.985 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1312369584 returned Deferred@502546523(state=PENDING, result=null, callback=(continuation of Deferred@2052811600 after retry RPC@1312369584), errback=(continuation of Deferred@2052811600 after retry RPC@1312369584)), so the following Deferred is getting paused: Deferred@2052811600(state=PAUSED, result=Deferred@502546523, callback=org.apache.kudu.client.AsyncKuduSession$2@45925080 -> passthrough -> wakeup thread pool-480-thread-41, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@42719729 -> wakeup thread pool-480-thread-41) 01:27:02.986 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.986 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.987 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=260), Traces: [0ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [260ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.993 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@2100879780 returned Deferred@1545077944(state=PENDING, result=null, callback=(continuation of Deferred@198789611 after retry RPC@2100879780), errback=(continuation of Deferred@198789611 after retry RPC@2100879780)), so the following Deferred is getting paused: Deferred@198789611(state=PAUSED, result=Deferred@1545077944, callback=org.apache.kudu.client.AsyncKuduSession$2@455f4f66 -> passthrough -> wakeup thread pool-480-thread-23, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7fd0a3a8 -> wakeup thread pool-480-thread-23) 01:27:02.994 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.994 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.995 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=269), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [269ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.995 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1668371758 returned Deferred@581357242(state=PENDING, result=null, callback=(continuation of Deferred@498958839 after retry RPC@1668371758), errback=(continuation of Deferred@498958839 after retry RPC@1668371758)), so the following Deferred is getting paused: Deferred@498958839(state=PAUSED, result=Deferred@581357242, callback=org.apache.kudu.client.AsyncKuduSession$2@678abc0b -> passthrough -> wakeup thread pool-480-thread-2, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@e3b4df1 -> wakeup thread pool-480-thread-2) 01:27:02.996 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.996 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.996 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=271), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [271ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.997 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@444367250 returned Deferred@1807685907(state=PENDING, result=null, callback=(continuation of Deferred@1508346507 after retry RPC@444367250), errback=(continuation of Deferred@1508346507 after retry RPC@444367250)), so the following Deferred is getting paused: Deferred@1508346507(state=PAUSED, result=Deferred@1807685907, callback=org.apache.kudu.client.AsyncKuduSession$2@d670083 -> passthrough -> wakeup thread pool-480-thread-27, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736960ac -> wakeup thread pool-480-thread-27) 01:27:02.997 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:02.998 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:02.998 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=273), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [272ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:02.999 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1611503699 returned Deferred@89912587(state=PENDING, result=null, callback=(continuation of Deferred@806198843 after retry RPC@1611503699), errback=(continuation of Deferred@806198843 after retry RPC@1611503699)), so the following Deferred is getting paused: Deferred@806198843(state=PAUSED, result=Deferred@89912587, callback=org.apache.kudu.client.AsyncKuduSession$2@859e496 -> passthrough -> wakeup thread pool-480-thread-11, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@49a3ace -> wakeup thread pool-480-thread-11) 01:27:02.999 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:03.000 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:03.001 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=267), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [265ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:03.003 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1580430587 returned Deferred@34854208(state=PENDING, result=null, callback=(continuation of Deferred@1160379584 after retry RPC@1580430587), errback=(continuation of Deferred@1160379584 after retry RPC@1580430587)), so the following Deferred is getting paused: Deferred@1160379584(state=PAUSED, result=Deferred@34854208, callback=org.apache.kudu.client.AsyncKuduSession$2@1030f2e1 -> passthrough -> wakeup thread pool-480-thread-43, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@55e1c8b0 -> wakeup thread pool-480-thread-43) 01:27:03.004 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:03.004 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:03.008 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=274), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [270ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:03.010 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@1237626521 returned Deferred@1124172957(state=PENDING, result=null, callback=(continuation of Deferred@1412547046 after retry RPC@1237626521), errback=(continuation of Deferred@1412547046 after retry RPC@1237626521)), so the following Deferred is getting paused: Deferred@1412547046(state=PAUSED, result=Deferred@1124172957, callback=org.apache.kudu.client.AsyncKuduSession$2@fce2798 -> passthrough -> wakeup thread pool-480-thread-45, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6392f09e -> wakeup thread pool-480-thread-45) 01:27:03.010 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:2118) Learned about tablet 352ac1600b824dd8bec48ce92773d445 for table 'testMultipleSessions' with partition [, ) 01:27:03.010 [DEBUG - New I/O worker #2143] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=352ac1600b824dd8bec48ce92773d445@[01faeb22d9e448ac83be7ed9e2e37b64(127.5.89.67:40645),14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)[L],42c3f16052ca4d0dbd066f5c63031d10(127.5.89.66:38592)]}] 01:27:03.010 [DEBUG - New I/O worker #2143] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=275), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.5.89.125:60269, [275ms] Sub RPC GetTableLocations: received response from server master-127.5.89.125:60269: OK, deferred=null) after lookup 01:27:03.013 [DEBUG - New I/O worker #2143] (Deferred.java:1330) callback=retry RPC@746104093 returned Deferred@1989963653(state=PENDING, result=null, callback=(continuation of Deferred@135291945 after retry RPC@746104093), errback=(continuation of Deferred@135291945 after retry RPC@746104093)), so the following Deferred is getting paused: Deferred@135291945(state=PAUSED, result=Deferred@1989963653, callback=org.apache.kudu.client.AsyncKuduSession$2@7c7263b5 -> passthrough -> wakeup thread pool-480-thread-46, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15cb616b -> wakeup thread pool-480-thread-46) 01:27:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:03.658552 6227 mvcc.cc:203] Tried to move safe_time back from 6351128263217508352 to 6351128258034720768. Current Snapshot: MvccSnapshot[committed={T|T < 6351128263217508352 or (T in {6351128263217508352})}] 01:27:12.072 [DEBUG - New I/O worker #2144] (Connection.java:429) [peer master-127.5.89.124:42353(127.5.89.124:42353)] encountered a read timeout; closing the channel 01:27:12.073 [DEBUG - New I/O worker #2144] (Connection.java:688) [peer master-127.5.89.124:42353(127.5.89.124:42353)] cleaning up while in state READY due to: [peer master-127.5.89.124:42353(127.5.89.124:42353)] encountered a read timeout; closing the channel 01:27:12.092 [DEBUG - New I/O worker #2145] (Connection.java:429) [peer master-127.5.89.126:45573(127.5.89.126:45573)] encountered a read timeout; closing the channel 01:27:12.092 [DEBUG - New I/O worker #2145] (Connection.java:688) [peer master-127.5.89.126:45573(127.5.89.126:45573)] cleaning up while in state READY due to: [peer master-127.5.89.126:45573(127.5.89.126:45573)] encountered a read timeout; closing the channel 01:27:13.012 [DEBUG - New I/O worker #2143] (Connection.java:429) [peer master-127.5.89.125:60269(127.5.89.125:60269)] encountered a read timeout; closing the channel 01:27:13.013 [DEBUG - New I/O worker #2143] (Connection.java:688) [peer master-127.5.89.125:60269(127.5.89.125:60269)] cleaning up while in state READY due to: [peer master-127.5.89.125:60269(127.5.89.125:60269)] encountered a read timeout; closing the channel 01:27:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:33.776793 5631 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 5619 01:27:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:33.778208 5631 debug-util.cc:397] Leaking SignalData structure 0x7b080007ae00 after lost signal to thread 5677 01:27:36.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:36.946228 5566 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 5554 01:27:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:36.948618 5566 debug-util.cc:397] Leaking SignalData structure 0x7b0800078400 after lost signal to thread 5612 01:27:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:44.848507 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3634, attempt_no=1}) took 1082ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":1808,"num_ops":1,"prepare.queue_time_us":778461,"prepare.run_cpu_time_us":16403,"prepare.run_wall_time_us":149963,"replication_time_us":122625,"row_lock_wait_count":1,"row_lock_wait_us":103762}]]} 01:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:44.896368 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3635, attempt_no=1}) took 1118ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":273,"num_ops":1,"prepare.queue_time_us":915776,"prepare.run_cpu_time_us":12456,"prepare.run_wall_time_us":133949,"replication_time_us":66199,"row_lock_wait_count":1,"row_lock_wait_us":113164}]]} 01:27:44.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:44.926825 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3636, attempt_no=1}) took 1137ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":294,"num_ops":1,"prepare.queue_time_us":1039201,"prepare.run_cpu_time_us":8200,"prepare.run_wall_time_us":71087,"replication_time_us":17667,"row_lock_wait_count":1,"row_lock_wait_us":52908}]]} 01:27:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:44.943114 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3637, attempt_no=1}) took 1138ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":2363,"num_ops":1,"prepare.queue_time_us":1095381,"prepare.run_cpu_time_us":2786,"prepare.run_wall_time_us":19779,"replication_time_us":15393,"row_lock_wait_count":1,"row_lock_wait_us":17024,"threads_started":1}]]} 01:27:44.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:44.970808 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3638, attempt_no=1}) took 1159ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":113,"num_ops":1,"prepare.queue_time_us":1107459,"prepare.run_cpu_time_us":3544,"prepare.run_wall_time_us":35203,"raft.queue_time_us":834,"raft.run_cpu_time_us":1857,"raft.run_wall_time_us":2883,"replication_time_us":29201,"row_lock_wait_count":1,"row_lock_wait_us":17044,"spinlock_wait_cycles":36877056}]]} 01:27:44.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:44.987151 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3639, attempt_no=1}) took 1163ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":195,"num_ops":1,"prepare.queue_time_us":1131695,"prepare.run_cpu_time_us":2195,"prepare.run_wall_time_us":15487,"replication_time_us":14724,"row_lock_wait_count":1,"row_lock_wait_us":13337,"spinlock_wait_cycles":110464}]]} 01:27:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.000800 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3640, attempt_no=1}) took 1163ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":87,"num_ops":1,"prepare.queue_time_us":1132618,"prepare.run_cpu_time_us":1798,"prepare.run_wall_time_us":16204,"replication_time_us":12292,"row_lock_wait_count":1,"row_lock_wait_us":14440}]]} 01:27:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.015592 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3641, attempt_no=1}) took 1169ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":83,"num_ops":1,"prepare.queue_time_us":1140575,"prepare.run_cpu_time_us":1611,"prepare.run_wall_time_us":17490,"replication_time_us":9456,"row_lock_wait_count":1,"row_lock_wait_us":15918,"spinlock_wait_cycles":123648}]]} 01:27:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.028863 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3642, attempt_no=1}) took 1172ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":175,"num_ops":1,"prepare.queue_time_us":1148335,"prepare.run_cpu_time_us":1439,"prepare.run_wall_time_us":12660,"replication_time_us":9760,"row_lock_wait_count":1,"row_lock_wait_us":11238}]]} 01:27:45.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.038372 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3643, attempt_no=1}) took 1171ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":172,"num_ops":1,"prepare.queue_time_us":1149693,"prepare.run_cpu_time_us":1458,"prepare.run_wall_time_us":11189,"replication_time_us":8066,"row_lock_wait_count":1,"row_lock_wait_us":9765,"spinlock_wait_cycles":35712}]]} 01:27:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.049018 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3644, attempt_no=1}) took 1171ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":182,"num_ops":1,"prepare.queue_time_us":1150880,"prepare.run_cpu_time_us":1484,"prepare.run_wall_time_us":9470,"replication_time_us":9299,"row_lock_wait_count":1,"row_lock_wait_us":8019,"spinlock_wait_cycles":41856}]]} 01:27:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.059756 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3645, attempt_no=1}) took 1173ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":179,"num_ops":1,"prepare.queue_time_us":1151983,"prepare.run_cpu_time_us":1524,"prepare.run_wall_time_us":10496,"replication_time_us":9149,"row_lock_wait_count":1,"row_lock_wait_us":9006,"spinlock_wait_cycles":52608}]]} 01:27:45.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.071065 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3646, attempt_no=1}) took 1175ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":671,"num_ops":1,"prepare.queue_time_us":1153176,"prepare.run_cpu_time_us":1582,"prepare.run_wall_time_us":10674,"replication_time_us":9218,"row_lock_wait_count":1,"row_lock_wait_us":9122,"spinlock_wait_cycles":55552}]]} 01:27:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.082516 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3647, attempt_no=1}) took 1177ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":719,"num_ops":1,"prepare.queue_time_us":1154264,"prepare.run_cpu_time_us":1559,"prepare.run_wall_time_us":11148,"replication_time_us":8991,"row_lock_wait_count":1,"row_lock_wait_us":9620,"spinlock_wait_cycles":62208}]]} 01:27:45.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.092692 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3648, attempt_no=1}) took 1176ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":193,"num_ops":1,"prepare.queue_time_us":1154539,"prepare.run_cpu_time_us":1463,"prepare.run_wall_time_us":10842,"replication_time_us":8204,"row_lock_wait_count":1,"row_lock_wait_us":9412,"spinlock_wait_cycles":403840}]]} 01:27:45.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.102365 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3649, attempt_no=1}) took 1176ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":194,"num_ops":1,"prepare.queue_time_us":1155613,"prepare.run_cpu_time_us":1569,"prepare.run_wall_time_us":10635,"replication_time_us":7823,"row_lock_wait_count":1,"row_lock_wait_us":9100,"spinlock_wait_cycles":70528}]]} 01:27:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.111913 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3650, attempt_no=1}) took 1176ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":112,"num_ops":1,"prepare.queue_time_us":1156745,"prepare.run_cpu_time_us":1488,"prepare.run_wall_time_us":9537,"replication_time_us":7276,"row_lock_wait_count":1,"row_lock_wait_us":8083,"spinlock_wait_cycles":46848}]]} 01:27:45.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.121914 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3652, attempt_no=1}) took 1166ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":111,"num_ops":1,"prepare.queue_time_us":1156253,"prepare.run_cpu_time_us":1302,"prepare.run_wall_time_us":1302,"replication_time_us":7384}]]} 01:27:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.122427 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3651, attempt_no=1}) took 1169ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":91,"num_ops":1,"prepare.queue_time_us":1148851,"prepare.run_cpu_time_us":1447,"prepare.run_wall_time_us":9431,"replication_time_us":8566,"row_lock_wait_count":1,"row_lock_wait_us":8035,"spinlock_wait_cycles":94080}]]} 01:27:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.136078 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3653, attempt_no=1}) took 1170ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":229,"num_ops":1,"prepare.queue_time_us":1148064,"prepare.run_cpu_time_us":1380,"prepare.run_wall_time_us":11972,"replication_time_us":9915,"row_lock_wait_count":1,"row_lock_wait_us":7859}]]} 01:27:45.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.143549 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3654, attempt_no=1}) took 1162ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":145,"num_ops":1,"prepare.queue_time_us":1143925,"prepare.run_cpu_time_us":1505,"prepare.run_wall_time_us":11079,"raft.queue_time_us":663,"raft.run_cpu_time_us":893,"raft.run_wall_time_us":893,"replication_time_us":5117,"row_lock_wait_count":1,"row_lock_wait_us":9592}]]} 01:27:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.155848 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3655, attempt_no=1}) took 1170ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":148,"num_ops":1,"prepare.queue_time_us":1150694,"prepare.run_cpu_time_us":1367,"prepare.run_wall_time_us":6963,"replication_time_us":7448,"row_lock_wait_count":1,"row_lock_wait_us":5614}]]} 01:27:45.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.165467 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3657, attempt_no=1}) took 1162ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":116,"num_ops":1,"prepare.queue_time_us":1149902,"prepare.run_cpu_time_us":1310,"prepare.run_wall_time_us":1310,"replication_time_us":9905}]]} 01:27:45.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.168059 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3656, attempt_no=1}) took 1168ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":117,"num_ops":1,"prepare.queue_time_us":1143416,"prepare.run_cpu_time_us":1435,"prepare.run_wall_time_us":9102,"replication_time_us":6834,"row_lock_wait_count":1,"row_lock_wait_us":7682,"spinlock_wait_cycles":65792}]]} 01:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.174089 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3658, attempt_no=1}) took 1157ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":84,"num_ops":1,"prepare.queue_time_us":1137280,"prepare.run_cpu_time_us":1352,"prepare.run_wall_time_us":8657,"replication_time_us":9695,"row_lock_wait_count":1,"row_lock_wait_us":7321}]]} 01:27:45.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.185047 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3659, attempt_no=1}) took 1165ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":565,"num_ops":1,"prepare.queue_time_us":1143349,"prepare.run_cpu_time_us":1360,"prepare.run_wall_time_us":2838,"replication_time_us":13424,"row_lock_wait_count":1,"row_lock_wait_us":1283,"spinlock_wait_cycles":37504}]]} 01:27:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.189930 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3660, attempt_no=1}) took 1163ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":207,"num_ops":1,"prepare.queue_time_us":1138822,"prepare.run_cpu_time_us":1409,"prepare.run_wall_time_us":15212,"replication_time_us":7292,"row_lock_wait_count":1,"row_lock_wait_us":13821}]]} 01:27:45.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.192360 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3661, attempt_no=1}) took 1161ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":97,"num_ops":1,"prepare.queue_time_us":1149761,"prepare.run_cpu_time_us":1363,"prepare.run_wall_time_us":1362,"replication_time_us":8295,"spinlock_wait_cycles":594688}]]} 01:27:45.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.205405 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3662, attempt_no=1}) took 1164ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":117,"num_ops":1,"prepare.queue_time_us":1141478,"prepare.run_cpu_time_us":1511,"prepare.run_wall_time_us":10079,"replication_time_us":11112,"row_lock_wait_count":1,"row_lock_wait_us":8587}]]} 01:27:45.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.206204 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3663, attempt_no=1}) took 1157ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":74,"num_ops":1,"prepare.queue_time_us":1143507,"prepare.run_cpu_time_us":1290,"prepare.run_wall_time_us":1490,"replication_time_us":9674,"spinlock_wait_cycles":50560}]]} 01:27:45.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.221156 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3664, attempt_no=1}) took 1164ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":115,"num_ops":1,"prepare.queue_time_us":1137285,"prepare.run_cpu_time_us":1331,"prepare.run_wall_time_us":11470,"replication_time_us":13493,"row_lock_wait_count":1,"row_lock_wait_us":10163}]]} 01:27:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.221444 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3665, attempt_no=1}) took 1155ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":176,"num_ops":1,"prepare.queue_time_us":1139340,"prepare.run_cpu_time_us":1400,"prepare.run_wall_time_us":1723,"replication_time_us":12374,"spinlock_wait_cycles":1010944}]]} 01:27:45.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.232141 6435 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3667, attempt_no=1}) took 1149ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":323,"num_ops":1,"prepare.queue_time_us":1138369,"prepare.run_cpu_time_us":1094,"prepare.run_wall_time_us":1094,"replication_time_us":7895,"spinlock_wait_cycles":70400}]]} 01:27:45.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.231990 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3666, attempt_no=1}) took 1156ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":288,"num_ops":1,"prepare.queue_time_us":1131830,"prepare.run_cpu_time_us":1388,"prepare.run_wall_time_us":13313,"replication_time_us":9113,"row_lock_wait_count":1,"row_lock_wait_us":11944}]]} 01:27:45.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.251139 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3668, attempt_no=1}) took 1157ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":172,"num_ops":1,"prepare.queue_time_us":1128229,"prepare.run_cpu_time_us":1235,"prepare.run_wall_time_us":9461,"replication_time_us":17746,"row_lock_wait_count":1,"row_lock_wait_us":8264,"spinlock_wait_cycles":79616}]]} 01:27:45.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.266137 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3669, attempt_no=1}) took 1162ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":203,"num_ops":1,"prepare.queue_time_us":1127429,"prepare.run_cpu_time_us":1402,"prepare.run_wall_time_us":19011,"replication_time_us":13455,"row_lock_wait_count":1,"row_lock_wait_us":17640,"spinlock_wait_cycles":79616}]]} 01:27:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.292312 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3670, attempt_no=1}) took 1177ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":208,"num_ops":1,"prepare.queue_time_us":1135888,"prepare.run_cpu_time_us":1438,"prepare.run_wall_time_us":14881,"replication_time_us":24682,"row_lock_wait_count":1,"row_lock_wait_us":13489,"spinlock_wait_cycles":71936}]]} 01:27:45.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.304492 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3671, attempt_no=1}) took 1180ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":198,"num_ops":1,"prepare.queue_time_us":1141782,"prepare.run_cpu_time_us":1574,"prepare.run_wall_time_us":26118,"replication_time_us":10737,"row_lock_wait_count":1,"row_lock_wait_us":24576,"spinlock_wait_cycles":44800}]]} 01:27:45.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.318801 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3672, attempt_no=1}) took 1184ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":219,"num_ops":1,"prepare.queue_time_us":1157918,"prepare.run_cpu_time_us":1510,"prepare.run_wall_time_us":12023,"replication_time_us":12799,"row_lock_wait_count":1,"row_lock_wait_us":10548,"spinlock_wait_cycles":83840}]]} 01:27:45.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.333184 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3673, attempt_no=1}) took 1192ms. Request Metrics: {"spinlock_wait_cycles":10496,"child_traces":[["txn",{"apply.queue_time_us":184,"num_ops":1,"prepare.queue_time_us":1163240,"prepare.run_cpu_time_us":1984,"prepare.run_wall_time_us":16996,"replication_time_us":13136,"row_lock_wait_count":1,"row_lock_wait_us":12682,"spinlock_wait_cycles":6027392}]]} 01:27:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.344882 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3674, attempt_no=1}) took 1192ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":222,"num_ops":1,"prepare.queue_time_us":1168813,"prepare.run_cpu_time_us":1461,"prepare.run_wall_time_us":11748,"replication_time_us":9962,"row_lock_wait_count":1,"row_lock_wait_us":10305,"spinlock_wait_cycles":62720}]]} 01:27:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.355653 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3675, attempt_no=1}) took 1194ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":169,"num_ops":1,"prepare.queue_time_us":1172214,"prepare.run_cpu_time_us":1414,"prepare.run_wall_time_us":13047,"replication_time_us":7732,"row_lock_wait_count":1,"row_lock_wait_us":11653,"spinlock_wait_cycles":18176}]]} 01:27:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.366166 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3676, attempt_no=1}) took 1194ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":249,"num_ops":1,"prepare.queue_time_us":1175269,"prepare.run_cpu_time_us":1387,"prepare.run_wall_time_us":11305,"replication_time_us":6194,"row_lock_wait_count":1,"row_lock_wait_us":9937}]]} 01:27:45.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.375998 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3677, attempt_no=1}) took 1191ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":77,"num_ops":1,"prepare.queue_time_us":1172905,"prepare.run_cpu_time_us":1487,"prepare.run_wall_time_us":8100,"replication_time_us":8647,"row_lock_wait_count":1,"row_lock_wait_us":6641,"spinlock_wait_cycles":89344}]]} 01:27:45.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.384452 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3678, attempt_no=1}) took 1188ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":98,"num_ops":1,"prepare.queue_time_us":1169579,"prepare.run_cpu_time_us":1351,"prepare.run_wall_time_us":9994,"replication_time_us":6799,"row_lock_wait_count":1,"row_lock_wait_us":8657}]]} 01:27:45.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.394170 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3679, attempt_no=1}) took 1188ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":222,"num_ops":1,"prepare.queue_time_us":1170732,"prepare.run_cpu_time_us":1439,"prepare.run_wall_time_us":9441,"replication_time_us":6823,"row_lock_wait_count":1,"row_lock_wait_us":8019}]]} 01:27:45.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.404031 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3680, attempt_no=1}) took 1188ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":130,"num_ops":1,"prepare.queue_time_us":1170991,"prepare.run_cpu_time_us":1543,"prepare.run_wall_time_us":9042,"replication_time_us":6948,"row_lock_wait_count":1,"row_lock_wait_us":7378}]]} 01:27:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.414692 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3681, attempt_no=1}) took 1188ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":320,"num_ops":1,"prepare.queue_time_us":1168676,"prepare.run_cpu_time_us":1485,"prepare.run_wall_time_us":9687,"replication_time_us":7322,"row_lock_wait_count":1,"row_lock_wait_us":8144,"spinlock_wait_cycles":35072}]]} 01:27:45.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.425681 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3682, attempt_no=1}) took 1187ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":151,"num_ops":1,"prepare.queue_time_us":1167140,"prepare.run_cpu_time_us":1498,"prepare.run_wall_time_us":10690,"raft.queue_time_us":109,"raft.run_cpu_time_us":346,"raft.run_wall_time_us":345,"replication_time_us":6432,"row_lock_wait_count":1,"row_lock_wait_us":8809,"spinlock_wait_cycles":1911552}]]} 01:27:45.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.434412 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3683, attempt_no=1}) took 1182ms. Request Metrics: {"spinlock_wait_cycles":191232,"child_traces":[["txn",{"apply.queue_time_us":104,"num_ops":1,"prepare.queue_time_us":1160976,"prepare.run_cpu_time_us":1430,"prepare.run_wall_time_us":8031,"replication_time_us":8503,"row_lock_wait_count":1,"row_lock_wait_us":6616}]]} 01:27:45.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.445163 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3684, attempt_no=1}) took 1156ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":96,"num_ops":1,"prepare.queue_time_us":1135579,"prepare.run_cpu_time_us":1431,"prepare.run_wall_time_us":9967,"replication_time_us":8521,"row_lock_wait_count":1,"row_lock_wait_us":8554}]]} 01:27:45.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.454741 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3685, attempt_no=1}) took 1148ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":136,"num_ops":1,"prepare.queue_time_us":1127320,"prepare.run_cpu_time_us":1445,"prepare.run_wall_time_us":11009,"replication_time_us":7376,"row_lock_wait_count":1,"row_lock_wait_us":9528}]]} 01:27:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.465283 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3686, attempt_no=1}) took 1141ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":122,"num_ops":1,"prepare.queue_time_us":1120695,"prepare.run_cpu_time_us":1476,"prepare.run_wall_time_us":10221,"replication_time_us":8436,"row_lock_wait_count":1,"row_lock_wait_us":7995}]]} 01:27:45.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.475529 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3687, attempt_no=1}) took 1122ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":109,"num_ops":1,"prepare.queue_time_us":1097187,"prepare.run_cpu_time_us":1536,"prepare.run_wall_time_us":10015,"replication_time_us":8352,"row_lock_wait_count":1,"row_lock_wait_us":7810}]]} 01:27:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.485579 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3688, attempt_no=1}) took 1125ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":121,"num_ops":1,"prepare.queue_time_us":1105562,"prepare.run_cpu_time_us":1553,"prepare.run_wall_time_us":10509,"replication_time_us":8371,"row_lock_wait_count":1,"row_lock_wait_us":8384}]]} 01:27:45.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.496390 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3689, attempt_no=1}) took 1124ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":106,"num_ops":1,"prepare.queue_time_us":1104090,"prepare.run_cpu_time_us":1537,"prepare.run_wall_time_us":10184,"replication_time_us":8437,"row_lock_wait_count":1,"row_lock_wait_us":7733}]]} 01:27:45.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.506119 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3690, attempt_no=1}) took 1108ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":105,"num_ops":1,"prepare.queue_time_us":1087066,"prepare.run_cpu_time_us":1712,"prepare.run_wall_time_us":12198,"replication_time_us":6789,"row_lock_wait_count":1,"row_lock_wait_us":7328}]]} 01:27:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.517096 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3691, attempt_no=1}) took 1108ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":263,"num_ops":1,"prepare.queue_time_us":1090285,"prepare.run_cpu_time_us":1408,"prepare.run_wall_time_us":8021,"replication_time_us":9114,"row_lock_wait_count":1,"row_lock_wait_us":5780}]]} 01:27:45.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.529060 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3692, attempt_no=1}) took 1105ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":232,"num_ops":1,"prepare.queue_time_us":1082628,"prepare.run_cpu_time_us":1580,"prepare.run_wall_time_us":10888,"replication_time_us":10107,"row_lock_wait_count":1,"row_lock_wait_us":8510}]]} 01:27:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:27:45.541971 6443 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=3693, attempt_no=1}) took 1005ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":121,"num_ops":1,"prepare.queue_time_us":937739,"prepare.run_cpu_time_us":1602,"prepare.run_wall_time_us":10564,"replication_time_us":9541,"row_lock_wait_count":1,"row_lock_wait_us":8837,"spinlock_wait_cycles":88448}]]} 01:27:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:52.738255 6001 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 5989 01:27:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:27:52.739817 6001 debug-util.cc:397] Leaking SignalData structure 0x7b08000ac4c0 after lost signal to thread 6107 01:28:01.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:01.778798 5718 debug-util.cc:397] Leaking SignalData structure 0x7b0800029080 after lost signal to thread 5693 01:28:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:01.780690 5718 debug-util.cc:397] Leaking SignalData structure 0x7b08000a5b20 after lost signal to thread 5835 01:28:02.053 [DEBUG - Time-limited test] (Deferred.java:1189) While joining Deferred@640752011(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$2@62d8aeed -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79b3e988 -> wakeup thread Time-limited test): interrupted 01:28:02.054 [DEBUG - New I/O worker #2146] (Connection.java:688) [peer 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277)] cleaning up while in state READY due to: connection disconnected 01:28:02.055 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.055 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.056 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.056 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.056 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.056 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.056 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.056 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.056 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.057 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.057 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.057 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.057 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.057 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.057 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.057 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.058 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.058 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.058 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.058374 5712 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.5.89.65:53105 (ReqId={client: b136b56aba97452aa63b43041ff0c241, seq_no=5255, attempt_no=1}) could send its response 01:28:02.054 [DEBUG - Test worker] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:28:02.058 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.059 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.061 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.077 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.077 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.078 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:02.064926 5477 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 5680 01:28:02.078 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.079 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.079 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.080 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.080 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.080 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.082 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.082 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.082 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.084 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.084 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.084 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.085 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.086 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.086 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.087 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.087 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.088 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.090 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.090 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.090 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.091 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.092 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.092 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.093 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.093 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.093 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.095 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.095 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.095 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.097 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.097 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.097 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.098 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.099 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.099 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.105 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.105 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.105 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.110 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.110 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.110 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.115 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.116 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.116 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.116037 5563 connection.cc:647] server connection from 127.5.89.65:47162 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.65:47162): Connection reset by peer (error 104) 01:28:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.118105 6000 connection.cc:647] server connection from 127.5.89.65:38096 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:28:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.119190 6000 connection.cc:320] Error closing socket: Network error: TlsSocket::Close: Broken pipe 01:28:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.118592 5495 connection.cc:647] server connection from 127.5.89.65:40235 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.65:40235): Connection reset by peer (error 104) 01:28:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.120056 5628 connection.cc:647] server connection from 127.5.89.65:39935 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.65:39935): Connection reset by peer (error 104) 01:28:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.120069 5855 connection.cc:647] server connection from 127.5.89.65:55268 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.65:55268): Connection reset by peer (error 104) 01:28:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:02.122261 5477 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 5838 01:28:02.124 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.124 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.125 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.130 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.130 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.131 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.136 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.136 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.137 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.144 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.144 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.145 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.149 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.150 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.150 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.155 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.155 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.155 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.160 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.160 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.161 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.165 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.166 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.166 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.170 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.171 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.171 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.175 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.176 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.176 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.181 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.181 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.181 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.187 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.187 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.188 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.192 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.193 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.193 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.197 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.198 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.198 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.202 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.202 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.203 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.207 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.208 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.208 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.213 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.213 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.213 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.182123 5563 connection.cc:647] server connection from 127.5.89.66:33984 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.66:33984): Connection reset by peer (error 104) 01:28:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.182772 5630 connection.cc:647] server connection from 127.5.89.66:54730 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.66:54730): Connection reset by peer (error 104) 01:28:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.183872 5497 connection.cc:647] server connection from 127.5.89.66:51086 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.66:51086): Connection reset by peer (error 104) 01:28:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:02.184001 5477 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 5971 01:28:02.218 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.218 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.219 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.223 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.223 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.224 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.228 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.228 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.229 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.233 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.233 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.234 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.239 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.239 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.239 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.244 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.244 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.244 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.252 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.252 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.253 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.246628 5626 connection.cc:647] server connection from 127.5.89.67:39043 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.67:39043): Connection reset by peer (error 104) 01:28:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.246923 5562 connection.cc:647] server connection from 127.5.89.67:51034 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.67:51034): Connection reset by peer (error 104) 01:28:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.248982 5493 connection.cc:647] server connection from 127.5.89.67:46482 recv error: Network error: failed to read from TLS socket (remote: 127.5.89.67:46482): Connection reset by peer (error 104) 01:28:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:02.249106 5477 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 5485 01:28:02.257 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.257 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.258 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.259 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.259 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.259 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.260 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.261 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.261 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.262 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.262 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.262 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.263 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.263 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.264 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.264 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.265 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.265 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.266 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.266 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.267 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.268 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.268 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.268 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.271 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.272 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.272 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.275 [INFO - New I/O worker #2146] (AsyncKuduClient.java:1970) Invalidating location 14bbf4194d1c467b9d518d02d40bb86e(127.5.89.65:34277) for tablet 352ac1600b824dd8bec48ce92773d445: connection disconnected 01:28:02.275 [DEBUG - New I/O worker #2146] (RemoteTablet.java:131) tablet 352ac1600b824dd8bec48ce92773d445 already removed ts 14bbf4194d1c467b9d518d02d40bb86e, size left is 2 01:28:02.276 [WARN - New I/O worker #2146] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:28:02.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:02.291038 5562 connection.cc:647] client connection to 127.5.89.126:45573 recv error: Network error: recv error from 0.0.0.0:0: Transport endpoint is not connected (error 107) 01:28:02.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:02.291280 5477 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 5547 01:28:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:02.312114 5477 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 5615 01:28:03.374 [ERROR - Test worker] (RetryRule.java:80) testMultipleSessions(org.apache.kudu.client.ITClientStress): failed attempt 1 org.junit.runners.model.TestTimedOutException: test timed out after 60000 milliseconds at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.stumbleupon.async.Deferred.doJoin(Deferred.java:1136) at com.stumbleupon.async.Deferred.join(Deferred.java:1019) at org.apache.kudu.client.KuduSession.apply(KuduSession.java:81) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:163) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:150) at org.apache.kudu.client.ITClientStress$1.run(ITClientStress.java:80) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:2022) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at org.apache.kudu.client.ITClientStress.runTasks(ITClientStress.java:86) at org.apache.kudu.client.ITClientStress.testMultipleSessions(ITClientStress.java:147) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 01:28:03.375 [INFO - Test worker] (KuduTestHarness.java:140) Creating a new MiniKuduCluster... 01:28:03.375 [INFO - Test worker] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 01:28:03.376 [INFO - Test worker] (KuduBinaryLocator.java:78) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 01:28:03.376 [INFO - Test worker] (MiniKuduCluster.java:187) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 01:28:03.379 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792" hms_mode: NONE mini_kdc_options { } } 01:28:03.658 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: 01:28:03.658 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: start_cluster { } 01:28:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.690991 6527 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:28:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:28:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/wal 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.254:34354 01:28:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.254 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/logs 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data/info.pb 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:28:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:28:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:28:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 with env {} 01:28:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:28:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:03.914008 6535 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:28:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:03.914543 6535 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:28:03.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:03.927093 6535 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:28:03.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:03.927275 6535 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:28:03.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:03.927392 6535 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:28:03.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:03.927541 6535 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:28:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.954561 6535 master_main.cc:73] Master server non-default flags: 01:28:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data 01:28:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/wal 01:28:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:28:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:03.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:28:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.254:34354 01:28:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:28:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:28:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data/info.pb 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.254 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:28:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/logs 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:28:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:28:03.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:28:03.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.971526 6535 master_main.cc:80] Initializing master server... 01:28:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.974896 6535 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 121378us 01:28:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.975919 6535 fs_manager.cc:263] Metadata directory not provided 01:28:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.976073 6535 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:28:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.976394 6535 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:28:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.976528 6535 server_base.cc:434] Attempting to create new FS layout instead 01:28:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.981631 6535 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data/instance: 01:28:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b0902a2690414c8f8f7852d328ad5f08" 01:28:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:03 on quasar-aecznm-1.vpc.cloudera.com" 01:28:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:03.982645 6535 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/wal/instance: 01:28:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b0902a2690414c8f8f7852d328ad5f08" 01:28:03.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:03 on quasar-aecznm-1.vpc.cloudera.com" 01:28:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.000982 6535 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.008s sys 0.008s 01:28:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.002552 6535 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:28:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.004335 6535 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:28:04.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.014459 6535 fs_manager.cc:419] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s 01:28:04.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.014613 6535 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/wal 01:28:04.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b0902a2690414c8f8f7852d328ad5f08" 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:03 on quasar-aecznm-1.vpc.cloudera.com" 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.014943 6535 fs_report.cc:352] FS layout report 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/wal 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/wal 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data/data 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:28:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:28:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:28:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:28:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:28:04.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.201795 6535 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:28:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.205250 6535 master_main.cc:83] Starting Master server... 01:28:04.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.336079 6535 rpc_server.cc:205] RPC server started. Bound to: 127.6.95.254:34354 01:28:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.337502 6535 webserver.cc:175] Starting webserver on 127.6.95.254:0 01:28:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.337651 6535 webserver.cc:186] Document root disabled 01:28:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.340777 6535 webserver.cc:313] Webserver started. Bound to: http://127.6.95.254:45925/ 01:28:04.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.341977 6535 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-0/data/info.pb 01:28:04.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.345019 6527 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 6535 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.346241 6527 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/wal 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.253:51702 01:28:04.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.253 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/logs 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data/info.pb 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 with env {} 01:28:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.346813 6596 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:28:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.364754 6596 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } has no permanent_uuid. Determining permanent_uuid... 01:28:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.397303 6596 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } has no permanent_uuid. Determining permanent_uuid... 01:28:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.400099 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.253:51702: Network error: Client connection negotiation failed: client connection to 127.6.95.253:51702: connect: Connection refused (error 111) 01:28:04.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.419683 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } attempt: 1 01:28:04.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.421535 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.253:51702: Network error: Client connection negotiation failed: client connection to 127.6.95.253:51702: connect: Connection refused (error 111) 01:28:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.469975 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } attempt: 2 01:28:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.471760 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.253:51702: Network error: Client connection negotiation failed: client connection to 127.6.95.253:51702: connect: Connection refused (error 111) 01:28:04.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.583227 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } attempt: 3 01:28:04.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.584923 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.253:51702: Network error: Client connection negotiation failed: client connection to 127.6.95.253:51702: connect: Connection refused (error 111) 01:28:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:28:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.596324 6597 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:28:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.596915 6597 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:28:04.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.609555 6597 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:28:04.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.609747 6597 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:28:04.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.610085 6597 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:28:04.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.610283 6597 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:28:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.638451 6597 master_main.cc:73] Master server non-default flags: 01:28:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data 01:28:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/wal 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.253:51702 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data/info.pb 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.253 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/logs 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:28:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:28:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:28:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:28:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:28:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:28:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:28:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:28:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.656982 6597 master_main.cc:80] Initializing master server... 01:28:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.661201 6597 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 121878us 01:28:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.662376 6597 fs_manager.cc:263] Metadata directory not provided 01:28:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.662587 6597 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:28:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.663234 6597 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:28:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.663411 6597 server_base.cc:434] Attempting to create new FS layout instead 01:28:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.669198 6597 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data/instance: 01:28:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "f11ed6e422304fc197605a2b3c8f5289" 01:28:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:04 on quasar-aecznm-1.vpc.cloudera.com" 01:28:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.670428 6597 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/wal/instance: 01:28:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "f11ed6e422304fc197605a2b3c8f5289" 01:28:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:04 on quasar-aecznm-1.vpc.cloudera.com" 01:28:04.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.689173 6597 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.010s sys 0.006s 01:28:04.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.690982 6597 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:28:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.692888 6597 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:28:04.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.703336 6597 fs_manager.cc:419] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s 01:28:04.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.703510 6597 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/wal 01:28:04.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "f11ed6e422304fc197605a2b3c8f5289" 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:04 on quasar-aecznm-1.vpc.cloudera.com" 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.703927 6597 fs_report.cc:352] FS layout report 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/wal 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/wal 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data/data 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:28:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:28:04.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:28:04.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:28:04.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:28:04.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:28:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.761337 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } attempt: 4 01:28:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:04.763366 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.253:51702: Network error: Client connection negotiation failed: client connection to 127.6.95.253:51702: connect: Connection refused (error 111) 01:28:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.941169 6597 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:28:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:04.945581 6597 master_main.cc:83] Starting Master server... 01:28:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.065981 6597 rpc_server.cc:205] RPC server started. Bound to: 127.6.95.253:51702 01:28:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.066817 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } attempt: 5 01:28:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.067965 6597 webserver.cc:175] Starting webserver on 127.6.95.253:0 01:28:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.068167 6597 webserver.cc:186] Document root disabled 01:28:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.071805 6597 webserver.cc:313] Webserver started. Bound to: http://127.6.95.253:35688/ 01:28:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.074759 6597 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-1/data/info.pb 01:28:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.081670 6661 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:28:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.083437 6527 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 6597 01:28:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.084758 6527 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/wal 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.252:47527 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.252 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/logs 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data/info.pb 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:28:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:28:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:28:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 with env {} 01:28:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.106006 6596 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.107614 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.109148 6661 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.126695 6661 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.136270 6661 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.138150 6661 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.146047 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 1 01:28:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.147622 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.157687 6661 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 1 01:28:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.159579 6661 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.196072 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 2 01:28:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.199709 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.208174 6661 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 2 01:28:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.210366 6661 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.300180 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 3 01:28:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.301955 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.322108 6661 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 3 01:28:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.324615 6661 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:28:05.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.331318 6662 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:28:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.332020 6662 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.348193 6662 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.348439 6662 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.348671 6662 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.348969 6662 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.381669 6662 master_main.cc:73] Master server non-default flags: 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/wal 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.252:47527 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data/info.pb 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.252 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/logs 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:28:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:28:05.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:28:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.400395 6662 master_main.cc:80] Initializing master server... 01:28:05.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.403821 6662 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 122378us 01:28:05.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.404866 6662 fs_manager.cc:263] Metadata directory not provided 01:28:05.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.405059 6662 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:28:05.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.405505 6662 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:28:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.405872 6662 server_base.cc:434] Attempting to create new FS layout instead 01:28:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.411209 6662 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data/instance: 01:28:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "d36a5dadac574bf09a14d7badf4464c4" 01:28:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:05 on quasar-aecznm-1.vpc.cloudera.com" 01:28:05.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.412269 6662 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/wal/instance: 01:28:05.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "d36a5dadac574bf09a14d7badf4464c4" 01:28:05.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:05 on quasar-aecznm-1.vpc.cloudera.com" 01:28:05.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.430956 6662 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.008s sys 0.008s 01:28:05.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.432610 6662 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:28:05.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.434473 6662 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:28:05.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.442364 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 4 01:28:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.444295 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.446235 6662 fs_manager.cc:419] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s 01:28:05.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.446413 6662 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/wal 01:28:05.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "d36a5dadac574bf09a14d7badf4464c4" 01:28:05.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:05 on quasar-aecznm-1.vpc.cloudera.com" 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.446771 6662 fs_report.cc:352] FS layout report 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/wal 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/wal 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data/data 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:28:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:28:05.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:28:05.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:28:05.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:28:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.501307 6661 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 4 01:28:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.504055 6661 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.643687 6662 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:28:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.647028 6662 master_main.cc:83] Starting Master server... 01:28:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.709826 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 5 01:28:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.711851 6596 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.6.95.252:47527: Network error: Client connection negotiation failed: client connection to 127.6.95.252:47527: connect: Connection refused (error 111) 01:28:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.779419 6662 rpc_server.cc:205] RPC server started. Bound to: 127.6.95.252:47527 01:28:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.780624 6662 webserver.cc:175] Starting webserver on 127.6.95.252:0 01:28:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.780812 6662 webserver.cc:186] Document root disabled 01:28:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.783470 6662 webserver.cc:313] Webserver started. Bound to: http://127.6.95.252:37689/ 01:28:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.784520 6662 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/master-2/data/info.pb 01:28:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.789736 6725 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:28:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.794713 6527 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 6662 01:28:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.796540 6527 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:28:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:28:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/wal 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.193:0 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.6.95.193 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.193 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/logs 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data/info.pb 01:28:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:28:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:28:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:28:05.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.807546 6661 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 5 01:28:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.816632 6725 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.844980 6725 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.850126 6661 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: Bootstrap starting. 01:28:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.856868 6661 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: No blocks or log segments found. Creating new log. 01:28:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.857612 6725 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } has no permanent_uuid. Determining permanent_uuid... 01:28:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.861922 6661 log.cc:525] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: Log is configured to *not* fsync() on all Append() calls 01:28:05.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.868599 6661 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: No bootstrap required, opened a new log 01:28:05.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.879055 6725 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: Bootstrap starting. 01:28:05.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.884186 6725 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: No blocks or log segments found. Creating new log. 01:28:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.887003 6725 log.cc:525] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: Log is configured to *not* fsync() on all Append() calls 01:28:05.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.893507 6661 raft_consensus.cc:340] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:05.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.894605 6661 raft_consensus.cc:366] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:28:05.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.895102 6661 raft_consensus.cc:705] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f11ed6e422304fc197605a2b3c8f5289, State: Initialized, Role: FOLLOWER 01:28:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.896533 6725 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: No bootstrap required, opened a new log 01:28:05.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.896270 6661 consensus_queue.cc:229] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.902078 6661 sys_catalog.cc:337] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.903916 6661 sys_catalog.cc:340] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: This master's current role is: FOLLOWER 01:28:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.905740 6661 sys_catalog.cc:424] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: configured and running, proceeding with master startup. 01:28:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.920856 6725 raft_consensus.cc:340] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.922271 6725 raft_consensus.cc:366] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:28:05.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.922899 6725 raft_consensus.cc:705] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d36a5dadac574bf09a14d7badf4464c4, State: Initialized, Role: FOLLOWER 01:28:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.924968 6725 consensus_queue.cc:229] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:05.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.934928 6725 sys_catalog.cc:337] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:05.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.949065 6725 sys_catalog.cc:340] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: This master's current role is: FOLLOWER 01:28:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.953542 6735 sys_catalog.cc:337] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.954836 6735 sys_catalog.cc:340] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.904250 6733 sys_catalog.cc:337] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.037194 6733 sys_catalog.cc:340] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:05.954687 6725 sys_catalog.cc:424] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: configured and running, proceeding with master startup. 01:28:06.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.078377 6662 master_main.cc:86] Master server successfully started. 01:28:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:28:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.088640 6726 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:28:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.089864 6726 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:28:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.091436 6736 catalog_manager.cc:1104] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:28:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.091753 6736 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:28:06.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:05.951884 6616 tablet.cc:1783] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:28:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.098842 6597 master_main.cc:86] Master server successfully started. 01:28:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.104957 6734 catalog_manager.cc:1104] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:28:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.105528 6734 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:28:06.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.116804 6726 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:28:06.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.117533 6726 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.95.193 01:28:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:06.151656 6681 tablet.cc:1783] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:28:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.160151 6735 raft_consensus.cc:472] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:28:06.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.160714 6735 raft_consensus.cc:494] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.164247 6735 leader_election.cc:296] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b0902a2690414c8f8f7852d328ad5f08 (127.6.95.254:34354), f11ed6e422304fc197605a2b3c8f5289 (127.6.95.253:51702) 01:28:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.165719 6637 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d36a5dadac574bf09a14d7badf4464c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f11ed6e422304fc197605a2b3c8f5289" is_pre_election: true 01:28:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.166838 6637 raft_consensus.cc:2346] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d36a5dadac574bf09a14d7badf4464c4 in term 0. 01:28:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.168272 6674 leader_election.cc:310] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d36a5dadac574bf09a14d7badf4464c4, f11ed6e422304fc197605a2b3c8f5289; no voters: 01:28:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.169418 6735 raft_consensus.cc:2640] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Leader pre-election won for term 1 01:28:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.169395 6726 tablet_server_main.cc:78] Tablet server non-default flags: 01:28:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data 01:28:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/wal 01:28:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.193:0 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data/info.pb 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.193 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.6.95.193 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/logs 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:28:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:28:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:28:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:28:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:28:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.171650 6735 raft_consensus.cc:472] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:28:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.172019 6735 raft_consensus.cc:2886] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 0 FOLLOWER]: Advancing to term 1 01:28:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.180379 6735 raft_consensus.cc:494] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.184195 6735 leader_election.cc:296] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [CANDIDATE]: Term 1 election: Requested vote from peers b0902a2690414c8f8f7852d328ad5f08 (127.6.95.254:34354), f11ed6e422304fc197605a2b3c8f5289 (127.6.95.253:51702) 01:28:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.184404 6637 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d36a5dadac574bf09a14d7badf4464c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f11ed6e422304fc197605a2b3c8f5289" 01:28:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.185073 6637 raft_consensus.cc:2886] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 0 FOLLOWER]: Advancing to term 1 01:28:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.192642 6637 raft_consensus.cc:2346] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d36a5dadac574bf09a14d7badf4464c4 in term 1. 01:28:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.194228 6674 leader_election.cc:310] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d36a5dadac574bf09a14d7badf4464c4, f11ed6e422304fc197605a2b3c8f5289; no voters: 01:28:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.195564 6735 raft_consensus.cc:2640] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 1 FOLLOWER]: Leader election won for term 1 01:28:06.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.210566 6735 raft_consensus.cc:667] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 1 LEADER]: Becoming Leader. State: Replica: d36a5dadac574bf09a14d7badf4464c4, State: Running, Role: LEADER 01:28:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.211846 6735 consensus_queue.cc:206] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.222108 6726 tablet_server_main.cc:85] Initializing tablet server... 01:28:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.227962 6726 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 122878us 01:28:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.227738 6743 sys_catalog.cc:337] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d36a5dadac574bf09a14d7badf4464c4. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.228727 6743 sys_catalog.cc:340] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: This master's current role is: LEADER 01:28:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.229341 6726 fs_manager.cc:263] Metadata directory not provided 01:28:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.229702 6726 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:28:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.230365 6726 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:28:06.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.231082 6726 server_base.cc:434] Attempting to create new FS layout instead 01:28:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.239550 6726 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data/instance: 01:28:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "1f2026756db9406383d1481ef7a1a9c3" 01:28:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:06 on quasar-aecznm-1.vpc.cloudera.com" 01:28:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.241096 6726 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/wal/instance: 01:28:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "1f2026756db9406383d1481ef7a1a9c3" 01:28:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:06 on quasar-aecznm-1.vpc.cloudera.com" 01:28:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.247655 6745 catalog_manager.cc:1046] Loading table and tablet metadata into memory... 01:28:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.251293 6596 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } attempt: 6 01:28:06.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.257920 6745 catalog_manager.cc:1057] Initializing Kudu internal certificate authority... 01:28:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.278142 6726 fs_manager.cc:503] Time spent creating directory manager: real 0.036s user 0.015s sys 0.011s 01:28:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.281564 6596 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08: Bootstrap starting. 01:28:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.283315 6726 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:28:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.287243 6596 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08: No blocks or log segments found. Creating new log. 01:28:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.287870 6726 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:28:06.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.290210 6596 log.cc:525] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08: Log is configured to *not* fsync() on all Append() calls 01:28:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.314096 6596 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08: No bootstrap required, opened a new log 01:28:06.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.320616 6726 fs_manager.cc:419] Time spent opening block manager: real 0.010s user 0.002s sys 0.006s 01:28:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.321090 6726 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/wal 01:28:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "1f2026756db9406383d1481ef7a1a9c3" 01:28:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:06 on quasar-aecznm-1.vpc.cloudera.com" 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.322010 6726 fs_report.cc:352] FS layout report 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/wal 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/wal 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data/data 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:28:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:28:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:28:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:28:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:28:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:28:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:28:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:28:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.345187 6596 raft_consensus.cc:340] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.346475 6596 raft_consensus.cc:366] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:28:06.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.347025 6596 raft_consensus.cc:705] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0902a2690414c8f8f7852d328ad5f08, State: Initialized, Role: FOLLOWER 01:28:06.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.348315 6596 consensus_queue.cc:229] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.355551 6751 sys_catalog.cc:337] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.357415 6751 sys_catalog.cc:340] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.409508 6637 raft_consensus.cc:1181] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 1 FOLLOWER]: Refusing update from remote peer d36a5dadac574bf09a14d7badf4464c4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:28:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.412542 6743 consensus_queue.cc:984] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:28:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.423588 6751 raft_consensus.cc:472] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:28:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.424109 6751 raft_consensus.cc:494] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } 01:28:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.426517 6751 leader_election.cc:296] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f11ed6e422304fc197605a2b3c8f5289 (127.6.95.253:51702), d36a5dadac574bf09a14d7badf4464c4 (127.6.95.252:47527) 01:28:06.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.428119 6636 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b0902a2690414c8f8f7852d328ad5f08" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f11ed6e422304fc197605a2b3c8f5289" is_pre_election: true 01:28:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.429268 6702 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b0902a2690414c8f8f7852d328ad5f08" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d36a5dadac574bf09a14d7badf4464c4" is_pre_election: true 01:28:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.430510 6702 raft_consensus.cc:2307] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [term 1 LEADER]: Leader pre-election vote request: Denying vote to candidate b0902a2690414c8f8f7852d328ad5f08 for term 1 because replica is either leader or believes a valid leader to be alive. 01:28:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.450031 6636 raft_consensus.cc:2321] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate b0902a2690414c8f8f7852d328ad5f08 for term 1 because replica is already servicing an update from a current leader or another vote. 01:28:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.452033 6733 sys_catalog.cc:337] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d36a5dadac574bf09a14d7badf4464c4. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.452874 6733 sys_catalog.cc:340] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.459889 6545 leader_election.cc:310] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0902a2690414c8f8f7852d328ad5f08; no voters: d36a5dadac574bf09a14d7badf4464c4, f11ed6e422304fc197605a2b3c8f5289 01:28:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.461035 6751 raft_consensus.cc:2886] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 0 FOLLOWER]: Advancing to term 1 01:28:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.464841 6755 mvcc.cc:203] Tried to move safe_time back from 6351128520316051456 to 6351128519536013312. Current Snapshot: MvccSnapshot[committed={T|T < 6351128520316051456}] 01:28:06.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.468775 6751 raft_consensus.cc:2588] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority 01:28:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.474287 6735 sys_catalog.cc:337] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.474444 6733 sys_catalog.cc:337] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.475142 6735 sys_catalog.cc:340] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: This master's current role is: LEADER 01:28:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.475244 6733 sys_catalog.cc:340] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.487686 6745 catalog_manager.cc:941] Generated new certificate authority record 01:28:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.490432 6745 catalog_manager.cc:1066] Loading token signing keys... 01:28:06.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.541956 6745 catalog_manager.cc:4095] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4: Generated new TSK 0 01:28:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.356997 6596 sys_catalog.cc:337] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.544234 6596 sys_catalog.cc:340] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.545910 6596 sys_catalog.cc:424] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: configured and running, proceeding with master startup. 01:28:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.545943 6573 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d36a5dadac574bf09a14d7badf4464c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b0902a2690414c8f8f7852d328ad5f08" is_pre_election: true 01:28:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.546175 6572 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d36a5dadac574bf09a14d7badf4464c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b0902a2690414c8f8f7852d328ad5f08" 01:28:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.548308 6571 raft_consensus.cc:1181] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 1 FOLLOWER]: Refusing update from remote peer d36a5dadac574bf09a14d7badf4464c4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:28:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.548990 6573 raft_consensus.cc:2321] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d36a5dadac574bf09a14d7badf4464c4 for term 1 because replica is already servicing an update from a current leader or another vote. 01:28:06.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.550138 6735 consensus_queue.cc:984] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:28:06.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.550541 6572 raft_consensus.cc:2321] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d36a5dadac574bf09a14d7badf4464c4 for term 1 because replica is already servicing an update from a current leader or another vote. 01:28:06.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.571398 6535 master_main.cc:86] Master server successfully started. 01:28:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.643071 6751 sys_catalog.cc:337] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d36a5dadac574bf09a14d7badf4464c4. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.644142 6751 sys_catalog.cc:340] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.644723 6768 sys_catalog.cc:337] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.645709 6768 sys_catalog.cc:340] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08 [sys.catalog]: This master's current role is: FOLLOWER 01:28:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.670439 6765 catalog_manager.cc:1102] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08: acquiring CA information for follower catalog manager: success 01:28:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.676687 6765 catalog_manager.cc:1130] T 00000000000000000000000000000000 P b0902a2690414c8f8f7852d328ad5f08: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 01:28:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.762312 6726 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:28:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.774392 6726 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:28:06.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.782848 6726 tablet_server_main.cc:90] Starting tablet server... 01:28:06.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.960558 6735 sys_catalog.cc:337] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d36a5dadac574bf09a14d7badf4464c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b0902a2690414c8f8f7852d328ad5f08" member_type: VOTER last_known_addr { host: "127.6.95.254" port: 34354 } } peers { permanent_uuid: "f11ed6e422304fc197605a2b3c8f5289" member_type: VOTER last_known_addr { host: "127.6.95.253" port: 51702 } } peers { permanent_uuid: "d36a5dadac574bf09a14d7badf4464c4" member_type: VOTER last_known_addr { host: "127.6.95.252" port: 47527 } } } 01:28:06.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:06.961557 6735 sys_catalog.cc:340] T 00000000000000000000000000000000 P d36a5dadac574bf09a14d7badf4464c4 [sys.catalog]: This master's current role is: LEADER 01:28:07.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.121903 6734 catalog_manager.cc:1102] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: acquiring CA information for follower catalog manager: success 01:28:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.126503 6734 catalog_manager.cc:1130] T 00000000000000000000000000000000 P f11ed6e422304fc197605a2b3c8f5289: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 01:28:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.352897 6726 rpc_server.cc:205] RPC server started. Bound to: 127.6.95.193:40409 01:28:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.354475 6726 webserver.cc:175] Starting webserver on 127.6.95.193:0 01:28:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.354621 6726 webserver.cc:186] Document root disabled 01:28:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.357924 6726 webserver.cc:313] Webserver started. Bound to: http://127.6.95.193:51379/ 01:28:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.359634 6726 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-0/data/info.pb 01:28:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.362978 6527 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 6726 01:28:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.364411 6527 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/wal 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.194:0 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.6.95.194 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.194 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/logs 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data/info.pb 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:28:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:28:07.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.407502 6726 tablet_server_main.cc:93] Tablet server successfully started. 01:28:07.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.458292 6883 heartbeater.cc:345] Connected to a master server at 127.6.95.252:47527 01:28:07.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.458742 6883 heartbeater.cc:426] Registering TS with master... 01:28:07.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.460736 6883 heartbeater.cc:475] Master 127.6.95.252:47527 requested a full tablet report, sending... 01:28:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.468008 6692 ts_manager.cc:98] Registered new tserver with Master: 1f2026756db9406383d1481ef7a1a9c3 (127.6.95.193:40409) 01:28:07.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.472401 6692 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.6.95.193:34386 01:28:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.459990 6881 heartbeater.cc:345] Connected to a master server at 127.6.95.253:51702 01:28:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.475385 6881 heartbeater.cc:426] Registering TS with master... 01:28:07.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.477025 6881 heartbeater.cc:475] Master 127.6.95.253:51702 requested a full tablet report, sending... 01:28:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.459611 6886 heartbeater.cc:345] Connected to a master server at 127.6.95.254:34354 01:28:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.480621 6886 heartbeater.cc:426] Registering TS with master... 01:28:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.481139 6886 heartbeater.cc:475] Master 127.6.95.254:34354 requested a full tablet report, sending... 01:28:07.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.483327 6627 ts_manager.cc:98] Registered new tserver with Master: 1f2026756db9406383d1481ef7a1a9c3 (127.6.95.193:40409) 01:28:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.485483 6563 ts_manager.cc:98] Registered new tserver with Master: 1f2026756db9406383d1481ef7a1a9c3 (127.6.95.193:40409) 01:28:07.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:28:07.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:07.761173 6882 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:28:07.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:07.762389 6882 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:28:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:07.783007 6882 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:28:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:07.783852 6882 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.95.194 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.839839 6882 tablet_server_main.cc:78] Tablet server non-default flags: 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/wal 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.194:0 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data/info.pb 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.194 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:28:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.6.95.194 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/logs 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:28:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:28:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.873663 6882 tablet_server_main.cc:85] Initializing tablet server... 01:28:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.880156 6882 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 123378us 01:28:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.881222 6882 fs_manager.cc:263] Metadata directory not provided 01:28:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.881615 6882 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:28:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.882278 6882 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:28:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.882688 6882 server_base.cc:434] Attempting to create new FS layout instead 01:28:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.889586 6882 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data/instance: 01:28:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "316c988cd2454ee19cb8a22f98a153e0" 01:28:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:07 on quasar-aecznm-1.vpc.cloudera.com" 01:28:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.891119 6882 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/wal/instance: 01:28:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "316c988cd2454ee19cb8a22f98a153e0" 01:28:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:07 on quasar-aecznm-1.vpc.cloudera.com" 01:28:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.920536 6882 fs_manager.cc:503] Time spent creating directory manager: real 0.029s user 0.014s sys 0.009s 01:28:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.923383 6882 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:28:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.925897 6882 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:28:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.946532 6882 fs_manager.cc:419] Time spent opening block manager: real 0.008s user 0.001s sys 0.005s 01:28:07.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.947096 6882 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/wal 01:28:07.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "316c988cd2454ee19cb8a22f98a153e0" 01:28:07.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:07 on quasar-aecznm-1.vpc.cloudera.com" 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:07.947829 6882 fs_report.cc:352] FS layout report 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/wal 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/wal 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data/data 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:28:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:28:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:08.272917 6882 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:28:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:08.281982 6882 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:28:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:08.288250 6882 tablet_server_main.cc:90] Starting tablet server... 01:28:08.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:08.479262 6883 heartbeater.cc:467] Master 127.6.95.252:47527 was elected leader, sending a full tablet report... 01:28:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.029523 6882 rpc_server.cc:205] RPC server started. Bound to: 127.6.95.194:44821 01:28:09.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.031435 6882 webserver.cc:175] Starting webserver on 127.6.95.194:0 01:28:09.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.031757 6882 webserver.cc:186] Document root disabled 01:28:09.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.038203 6882 webserver.cc:313] Webserver started. Bound to: http://127.6.95.194:50191/ 01:28:09.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.039515 6882 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-1/data/info.pb 01:28:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.048753 6527 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 6882 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.050184 6527 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/wal 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.195:0 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.6.95.195 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.195 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:09.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/logs 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data/info.pb 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:28:09.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:28:09.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.104053 6882 tablet_server_main.cc:93] Tablet server successfully started. 01:28:09.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.137462 7013 heartbeater.cc:345] Connected to a master server at 127.6.95.252:47527 01:28:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.137917 7013 heartbeater.cc:426] Registering TS with master... 01:28:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.139576 7013 heartbeater.cc:475] Master 127.6.95.252:47527 requested a full tablet report, sending... 01:28:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.142472 6692 ts_manager.cc:98] Registered new tserver with Master: 316c988cd2454ee19cb8a22f98a153e0 (127.6.95.194:44821) 01:28:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.144701 6692 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.6.95.194:54523 01:28:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.138625 7012 heartbeater.cc:345] Connected to a master server at 127.6.95.253:51702 01:28:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.159350 7012 heartbeater.cc:426] Registering TS with master... 01:28:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.159860 7012 heartbeater.cc:475] Master 127.6.95.253:51702 requested a full tablet report, sending... 01:28:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.138931 7017 heartbeater.cc:345] Connected to a master server at 127.6.95.254:34354 01:28:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.162977 6627 ts_manager.cc:98] Registered new tserver with Master: 316c988cd2454ee19cb8a22f98a153e0 (127.6.95.194:44821) 01:28:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.163322 7017 heartbeater.cc:426] Registering TS with master... 01:28:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.163731 7017 heartbeater.cc:475] Master 127.6.95.254:34354 requested a full tablet report, sending... 01:28:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.166800 6563 ts_manager.cc:98] Registered new tserver with Master: 316c988cd2454ee19cb8a22f98a153e0 (127.6.95.194:44821) 01:28:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:28:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:09.291817 7016 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:28:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:09.292383 7016 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:28:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:09.303045 7016 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:28:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:09.303587 7016 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.95.195 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.323017 7016 tablet_server_main.cc:78] Tablet server non-default flags: 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/wal 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.6.95.195:0 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data/info.pb 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.6.95.195 01:28:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.6.95.254:34354,127.6.95.253:51702,127.6.95.252:47527 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.6.95.195 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/logs 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:28:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:28:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.340925 7016 tablet_server_main.cc:85] Initializing tablet server... 01:28:09.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.343590 7016 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 124378us 01:28:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.344215 7016 fs_manager.cc:263] Metadata directory not provided 01:28:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.344385 7016 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:28:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.344898 7016 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:28:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.345050 7016 server_base.cc:434] Attempting to create new FS layout instead 01:28:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.348975 7016 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data/instance: 01:28:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "5badfe537b704c53b985d5f918f31d1a" 01:28:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:09 on quasar-aecznm-1.vpc.cloudera.com" 01:28:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.350004 7016 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/wal/instance: 01:28:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "5badfe537b704c53b985d5f918f31d1a" 01:28:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:09 on quasar-aecznm-1.vpc.cloudera.com" 01:28:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.368070 7016 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.011s sys 0.005s 01:28:09.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.369866 7016 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:28:09.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.370548 7016 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:28:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.380095 7016 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s 01:28:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.380275 7016 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/wal 01:28:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "5badfe537b704c53b985d5f918f31d1a" 01:28:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:28:09 on quasar-aecznm-1.vpc.cloudera.com" 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.380642 7016 fs_report.cc:352] FS layout report 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/wal 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/wal 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data/data 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:28:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:28:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.634788 7016 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:28:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.641788 7016 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:28:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.643926 7016 tablet_server_main.cc:90] Starting tablet server... 01:28:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.790969 7016 rpc_server.cc:205] RPC server started. Bound to: 127.6.95.195:38961 01:28:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.792744 7016 webserver.cc:175] Starting webserver on 127.6.95.195:0 01:28:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.792953 7016 webserver.cc:186] Document root disabled 01:28:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.795868 7016 webserver.cc:313] Webserver started. Bound to: http://127.6.95.195:37843/ 01:28:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.797163 7016 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8011669941494113792/ts-2/data/info.pb 01:28:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.804462 6527 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 7016 01:28:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.816627 7016 tablet_server_main.cc:93] Tablet server successfully started. 01:28:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.885697 7144 heartbeater.cc:345] Connected to a master server at 127.6.95.252:47527 01:28:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.886267 7144 heartbeater.cc:426] Registering TS with master... 01:28:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.888092 7144 heartbeater.cc:475] Master 127.6.95.252:47527 requested a full tablet report, sending... 01:28:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.891433 6692 ts_manager.cc:98] Registered new tserver with Master: 5badfe537b704c53b985d5f918f31d1a (127.6.95.195:38961) 01:28:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.894083 6692 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.6.95.195:47396 01:28:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.895987 7145 heartbeater.cc:345] Connected to a master server at 127.6.95.254:34354 01:28:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.896334 7145 heartbeater.cc:426] Registering TS with master... 01:28:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.897083 7145 heartbeater.cc:475] Master 127.6.95.254:34354 requested a full tablet report, sending... 01:28:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.900107 6563 ts_manager.cc:98] Registered new tserver with Master: 5badfe537b704c53b985d5f918f31d1a (127.6.95.195:38961) 01:28:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.887289 7143 heartbeater.cc:345] Connected to a master server at 127.6.95.253:51702 01:28:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.905647 7143 heartbeater.cc:426] Registering TS with master... 01:28:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.906117 7143 heartbeater.cc:475] Master 127.6.95.253:51702 requested a full tablet report, sending... 01:28:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.908777 6627 ts_manager.cc:98] Registered new tserver with Master: 5badfe537b704c53b985d5f918f31d1a (127.6.95.195:38961) 01:28:09.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.913810 6527 external_mini_cluster.cc:623] 3 TS(s) registered with all masters 01:28:09.914 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: 01:28:09.914 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: get_masters { } 01:28:09.915 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.6.95.254" port: 34354 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.6.95.253" port: 51702 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.6.95.252" port: 47527 } } } 01:28:09.916 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: get_tservers { } 01:28:09.921 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.6.95.193" port: 40409 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.6.95.194" port: 44821 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.6.95.195" port: 38961 } } } 01:28:09.922 [INFO - Test worker] (KuduTestHarness.java:142) Creating a new Kudu client... 01:28:09.925 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:28:09.932 [DEBUG - New I/O worker #2153] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.6.95.252 01:28:09.932 [DEBUG - New I/O worker #2152] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.6.95.253 01:28:09.932 [DEBUG - New I/O worker #2154] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.6.95.254 01:28:09.953 [DEBUG - New I/O worker #2153] (Negotiator.java:764) Authenticated connection [id: 0x67763373, /127.6.95.252:46521 => /127.6.95.252:47527] using SASL/PLAIN 01:28:09.955 [DEBUG - New I/O worker #2152] (Negotiator.java:764) Authenticated connection [id: 0xfa84e284, /127.6.95.253:57628 => /127.6.95.253:51702] using SASL/PLAIN 01:28:09.957 [DEBUG - New I/O worker #2154] (Negotiator.java:764) Authenticated connection [id: 0x44af44b9, /127.6.95.254:46397 => /127.6.95.254:34354] using SASL/PLAIN 01:28:09.961 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:28:09.962 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=35), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.95.253:51702, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.95.252:47527, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.95.254:34354, [35ms] Sub RPC ConnectToMaster: received response from server master-127.6.95.254:34354: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.6.95.252:47527: OK, deferred=null) after lookup 01:28:09.962 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=retry RPC@1983657490 returned Deferred@859743921(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after retry RPC@1983657490), errback=(continuation of Deferred@1833947897 after retry RPC@1983657490)), so the following Deferred is getting paused: Deferred@1833947897(state=PAUSED, result=Deferred@859743921, callback=org.apache.kudu.client.AsyncKuduClient$1@2c75426 -> org.apache.kudu.client.AsyncKuduClient$2@155c7eca -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:09.964558 6692 catalog_manager.cc:1386] Servicing CreateTable request from {username='jenkins'} at 127.6.95.252:46521: 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "testMultipleSessions" 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) schema { 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: true 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:28:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column1_i" 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column2_i" 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column3_s" 01:28:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: STRING 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: true 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) encoding: DICT_ENCODING 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) compression: LZ4 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 4096 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column4_b" 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: BOOL 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) partition_schema { 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) range_schema { 01:28:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:28:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:28:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:28:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:09.982462 6692 catalog_manager.cc:1537] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table testMultipleSessions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 01:28:10.016 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@2c75426@46617638 returned Deferred@1393646078(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@405a917f -> (continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$1@2c75426@46617638), errback=passthrough -> (continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$1@2c75426@46617638)), so the following Deferred is getting paused: Deferred@1833947897(state=PAUSED, result=Deferred@1393646078, callback=org.apache.kudu.client.AsyncKuduClient$2@155c7eca -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:28:10.020 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:761) Opened table 5e47708aa0a043e19630671348bc6079 01:28:10.020 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306)), so the following Deferred is getting paused: Deferred@1833947897(state=PAUSED, result=Deferred@2013563702, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 01:28:10.030 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498)), so the following Deferred is getting paused: Deferred@1465869902(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.048475 7080 tablet_service.cc:870] Processing CreateTablet for tablet 8798d444603e4dd7943589ff83e108f8 (table=testMultipleSessions [id=5e47708aa0a043e19630671348bc6079]), partition=RANGE (key) PARTITION UNBOUNDED 01:28:10.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.050446 7080 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 8798d444603e4dd7943589ff83e108f8. 1 dirs total, 0 dirs full, 0 dirs failed 01:28:10.050 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371)), so the following Deferred is getting paused: Deferred@521910989(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.056958 6949 tablet_service.cc:870] Processing CreateTablet for tablet 8798d444603e4dd7943589ff83e108f8 (table=testMultipleSessions [id=5e47708aa0a043e19630671348bc6079]), partition=RANGE (key) PARTITION UNBOUNDED 01:28:10.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.058054 6949 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 8798d444603e4dd7943589ff83e108f8. 1 dirs total, 0 dirs full, 0 dirs failed 01:28:10.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.058122 6818 tablet_service.cc:870] Processing CreateTablet for tablet 8798d444603e4dd7943589ff83e108f8 (table=testMultipleSessions [id=5e47708aa0a043e19630671348bc6079]), partition=RANGE (key) PARTITION UNBOUNDED 01:28:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.059306 6818 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 8798d444603e4dd7943589ff83e108f8. 1 dirs total, 0 dirs full, 0 dirs failed 01:28:10.070 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652)), so the following Deferred is getting paused: Deferred@2102035973(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.074626 6949 ts_tablet_manager.cc:1175] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Registered tablet (data state: TABLET_DATA_READY) 01:28:10.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.076766 7184 ts_tablet_manager.cc:1030] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Bootstrapping tablet 01:28:10.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.077845 7184 tablet_bootstrap.cc:438] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Bootstrap starting. 01:28:10.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.081243 7184 tablet_bootstrap.cc:592] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: No blocks or log segments found. Creating new log. 01:28:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.083062 7184 log.cc:525] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Log is configured to *not* fsync() on all Append() calls 01:28:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.084108 7080 ts_tablet_manager.cc:1175] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Registered tablet (data state: TABLET_DATA_READY) 01:28:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.085367 6818 ts_tablet_manager.cc:1175] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Registered tablet (data state: TABLET_DATA_READY) 01:28:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.087057 7184 tablet_bootstrap.cc:438] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: No bootstrap required, opened a new log 01:28:10.089 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827)), so the following Deferred is getting paused: Deferred@720336960(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.087577 7184 ts_tablet_manager.cc:1047] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Time spent bootstrapping tablet: real 0.010s user 0.006s sys 0.003s 01:28:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.088407 7186 ts_tablet_manager.cc:1030] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Bootstrapping tablet 01:28:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.089210 7186 tablet_bootstrap.cc:438] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Bootstrap starting. 01:28:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.091430 7187 ts_tablet_manager.cc:1030] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Bootstrapping tablet 01:28:10.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.092187 7187 tablet_bootstrap.cc:438] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Bootstrap starting. 01:28:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.095990 7187 tablet_bootstrap.cc:592] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: No blocks or log segments found. Creating new log. 01:28:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.096839 7186 tablet_bootstrap.cc:592] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: No blocks or log segments found. Creating new log. 01:28:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.097635 7187 log.cc:525] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Log is configured to *not* fsync() on all Append() calls 01:28:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.099567 7186 log.cc:525] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Log is configured to *not* fsync() on all Append() calls 01:28:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.101505 7187 tablet_bootstrap.cc:438] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: No bootstrap required, opened a new log 01:28:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.102321 7187 ts_tablet_manager.cc:1047] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.003s 01:28:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.104904 7186 tablet_bootstrap.cc:438] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: No bootstrap required, opened a new log 01:28:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.105917 7186 ts_tablet_manager.cc:1047] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Time spent bootstrapping tablet: real 0.017s user 0.011s sys 0.004s 01:28:10.109 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827) -> (continuation of Deferred@226658887 after org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827) -> (continuation of Deferred@226658887 after org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238)), so the following Deferred is getting paused: Deferred@226658887(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.111024 7184 raft_consensus.cc:340] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.112218 7184 raft_consensus.cc:366] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:28:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.112507 7184 raft_consensus.cc:705] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 316c988cd2454ee19cb8a22f98a153e0, State: Initialized, Role: FOLLOWER 01:28:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.113658 7184 consensus_queue.cc:229] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.117548 7013 heartbeater.cc:467] Master 127.6.95.252:47527 was elected leader, sending a full tablet report... 01:28:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.121181 7184 ts_tablet_manager.cc:1075] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Time spent starting tablet: real 0.033s user 0.025s sys 0.003s 01:28:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.124745 7187 raft_consensus.cc:340] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.126363 7187 raft_consensus.cc:366] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:28:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.126971 7187 raft_consensus.cc:705] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1f2026756db9406383d1481ef7a1a9c3, State: Initialized, Role: FOLLOWER 01:28:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.128379 7187 consensus_queue.cc:229] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.130102 7186 raft_consensus.cc:340] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.131408 7186 raft_consensus.cc:366] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:28:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.131858 7186 raft_consensus.cc:705] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5badfe537b704c53b985d5f918f31d1a, State: Initialized, Role: FOLLOWER 01:28:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.133795 7186 consensus_queue.cc:229] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.137318 7192 raft_consensus.cc:472] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:28:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.138468 7187 ts_tablet_manager.cc:1075] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Time spent starting tablet: real 0.036s user 0.024s sys 0.002s 01:28:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.138125 7192 raft_consensus.cc:494] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.140972 7192 leader_election.cc:296] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5badfe537b704c53b985d5f918f31d1a (127.6.95.195:38961), 316c988cd2454ee19cb8a22f98a153e0 (127.6.95.194:44821) 01:28:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.145433 7144 heartbeater.cc:467] Master 127.6.95.252:47527 was elected leader, sending a full tablet report... 01:28:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.148269 7186 ts_tablet_manager.cc:1075] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Time spent starting tablet: real 0.042s user 0.022s sys 0.007s 01:28:10.152 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@665bdbd6@1717296086 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827) -> (continuation of Deferred@226658887 after org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238) -> (continuation of Deferred@925340197 after org.apache.kudu.client.AsyncKuduClient$15@665bdbd6@1717296086), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827) -> (continuation of Deferred@226658887 after org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238) -> (continuation of Deferred@925340197 after org.apache.kudu.client.AsyncKuduClient$15@665bdbd6@1717296086)), so the following Deferred is getting paused: Deferred@925340197(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:10.157140 6887 tablet.cc:1783] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:28:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.166781 7100 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8798d444603e4dd7943589ff83e108f8" candidate_uuid: "1f2026756db9406383d1481ef7a1a9c3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5badfe537b704c53b985d5f918f31d1a" is_pre_election: true 01:28:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.167616 7100 raft_consensus.cc:2346] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f2026756db9406383d1481ef7a1a9c3 in term 0. 01:28:10.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.168480 6969 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8798d444603e4dd7943589ff83e108f8" candidate_uuid: "1f2026756db9406383d1481ef7a1a9c3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "316c988cd2454ee19cb8a22f98a153e0" is_pre_election: true 01:28:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.169311 6969 raft_consensus.cc:2346] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f2026756db9406383d1481ef7a1a9c3 in term 0. 01:28:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.169555 6753 leader_election.cc:310] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f2026756db9406383d1481ef7a1a9c3, 5badfe537b704c53b985d5f918f31d1a; no voters: 01:28:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.170516 7191 raft_consensus.cc:2640] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Leader pre-election won for term 1 01:28:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.170812 7191 raft_consensus.cc:472] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:28:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.171104 7191 raft_consensus.cc:2886] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 0 FOLLOWER]: Advancing to term 1 01:28:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.174921 7191 raft_consensus.cc:494] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.176237 7191 leader_election.cc:296] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [CANDIDATE]: Term 1 election: Requested vote from peers 5badfe537b704c53b985d5f918f31d1a (127.6.95.195:38961), 316c988cd2454ee19cb8a22f98a153e0 (127.6.95.194:44821) 01:28:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.177551 7100 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8798d444603e4dd7943589ff83e108f8" candidate_uuid: "1f2026756db9406383d1481ef7a1a9c3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5badfe537b704c53b985d5f918f31d1a" 01:28:10.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.177692 6969 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "8798d444603e4dd7943589ff83e108f8" candidate_uuid: "1f2026756db9406383d1481ef7a1a9c3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "316c988cd2454ee19cb8a22f98a153e0" 01:28:10.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.177947 7100 raft_consensus.cc:2886] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 0 FOLLOWER]: Advancing to term 1 01:28:10.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.178170 6969 raft_consensus.cc:2886] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 0 FOLLOWER]: Advancing to term 1 01:28:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.181813 6969 raft_consensus.cc:2346] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f2026756db9406383d1481ef7a1a9c3 in term 1. 01:28:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.181854 7100 raft_consensus.cc:2346] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f2026756db9406383d1481ef7a1a9c3 in term 1. 01:28:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.183074 6753 leader_election.cc:310] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f2026756db9406383d1481ef7a1a9c3, 316c988cd2454ee19cb8a22f98a153e0; no voters: 01:28:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.183862 7191 raft_consensus.cc:2640] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 1 FOLLOWER]: Leader election won for term 1 01:28:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.184202 7191 raft_consensus.cc:667] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [term 1 LEADER]: Becoming Leader. State: Replica: 1f2026756db9406383d1481ef7a1a9c3, State: Running, Role: LEADER 01:28:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.185201 7191 consensus_queue.cc:206] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } } 01:28:10.189 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@190bfcdd@420216029 returned Deferred@2013563702(state=PENDING, result=null, callback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827) -> (continuation of Deferred@226658887 after org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238) -> (continuation of Deferred@925340197 after org.apache.kudu.client.AsyncKuduClient$15@665bdbd6@1717296086) -> (continuation of Deferred@12750089 after org.apache.kudu.client.AsyncKuduClient$15@190bfcdd@420216029), errback=(continuation of Deferred@1833947897 after org.apache.kudu.client.AsyncKuduClient$2@155c7eca@358383306) -> (continuation of Deferred@1465869902 after org.apache.kudu.client.AsyncKuduClient$15@e5810d2@240652498) -> (continuation of Deferred@521910989 after org.apache.kudu.client.AsyncKuduClient$15@439ac4b3@1134216371) -> (continuation of Deferred@2102035973 after org.apache.kudu.client.AsyncKuduClient$15@510be77c@1359734652) -> (continuation of Deferred@720336960 after org.apache.kudu.client.AsyncKuduClient$15@68e07a43@1759541827) -> (continuation of Deferred@226658887 after org.apache.kudu.client.AsyncKuduClient$15@24514a66@609307238) -> (continuation of Deferred@925340197 after org.apache.kudu.client.AsyncKuduClient$15@665bdbd6@1717296086) -> (continuation of Deferred@12750089 after org.apache.kudu.client.AsyncKuduClient$15@190bfcdd@420216029)), so the following Deferred is getting paused: Deferred@12750089(state=PAUSED, result=Deferred@2013563702, callback=, errback=) 01:28:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.197065 6692 catalog_manager.cc:3791] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 reported cstate change: term changed from 0 to 1, leader changed from to 1f2026756db9406383d1481ef7a1a9c3 (127.6.95.193). New cstate: current_term: 1 leader_uuid: "1f2026756db9406383d1481ef7a1a9c3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1f2026756db9406383d1481ef7a1a9c3" member_type: VOTER last_known_addr { host: "127.6.95.193" port: 40409 } health_report { overall_health: HEALTHY } } } 01:28:10.313 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:2118) Learned about tablet 8798d444603e4dd7943589ff83e108f8 for table 'testMultipleSessions' with partition [, ) 01:28:10.313 [DEBUG - New I/O worker #2153] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8798d444603e4dd7943589ff83e108f8@[1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409)[L],316c988cd2454ee19cb8a22f98a153e0(127.6.95.194:44821),5badfe537b704c53b985d5f918f31d1a(127.6.95.195:38961)]}] 01:28:10.313 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.95.252:47527, [2ms] Sub RPC GetTableLocations: received response from server master-127.6.95.252:47527: OK, deferred=null) after lookup 01:28:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:10.314635 7147 tablet.cc:1783] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:28:10.320 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=retry RPC@947876780 returned Deferred@859990076(state=PENDING, result=null, callback=(continuation of Deferred@1969854365 after retry RPC@947876780), errback=(continuation of Deferred@1969854365 after retry RPC@947876780)), so the following Deferred is getting paused: Deferred@1969854365(state=PAUSED, result=Deferred@859990076, callback=org.apache.kudu.client.AsyncKuduSession$2@6157e6fc -> passthrough -> wakeup thread pool-483-thread-1, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@22f61b69 -> wakeup thread pool-483-thread-1) 01:28:10.321 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:2118) Learned about tablet 8798d444603e4dd7943589ff83e108f8 for table 'testMultipleSessions' with partition [, ) 01:28:10.321 [DEBUG - New I/O worker #2153] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8798d444603e4dd7943589ff83e108f8@[1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409)[L],316c988cd2454ee19cb8a22f98a153e0(127.6.95.194:44821),5badfe537b704c53b985d5f918f31d1a(127.6.95.195:38961)]}] 01:28:10.321 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=10), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.95.252:47527, [10ms] Sub RPC GetTableLocations: received response from server master-127.6.95.252:47527: OK, deferred=null) after lookup 01:28:10.322 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=retry RPC@47551619 returned Deferred@1001995835(state=PENDING, result=null, callback=(continuation of Deferred@1694977974 after retry RPC@47551619), errback=(continuation of Deferred@1694977974 after retry RPC@47551619)), so the following Deferred is getting paused: Deferred@1694977974(state=PAUSED, result=Deferred@1001995835, callback=org.apache.kudu.client.AsyncKuduSession$2@77b83d2c -> passthrough -> wakeup thread pool-483-thread-2, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5967ded4 -> wakeup thread pool-483-thread-2) 01:28:10.322 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:2118) Learned about tablet 8798d444603e4dd7943589ff83e108f8 for table 'testMultipleSessions' with partition [, ) 01:28:10.322 [DEBUG - New I/O worker #2153] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8798d444603e4dd7943589ff83e108f8@[1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409)[L],316c988cd2454ee19cb8a22f98a153e0(127.6.95.194:44821),5badfe537b704c53b985d5f918f31d1a(127.6.95.195:38961)]}] 01:28:10.322 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=10), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.95.252:47527, [10ms] Sub RPC GetTableLocations: received response from server master-127.6.95.252:47527: OK, deferred=null) after lookup 01:28:10.323 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=retry RPC@727546661 returned Deferred@1954085639(state=PENDING, result=null, callback=(continuation of Deferred@1109391865 after retry RPC@727546661), errback=(continuation of Deferred@1109391865 after retry RPC@727546661)), so the following Deferred is getting paused: Deferred@1109391865(state=PAUSED, result=Deferred@1954085639, callback=org.apache.kudu.client.AsyncKuduSession$2@25e91e10 -> passthrough -> wakeup thread pool-483-thread-3, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65adcddb -> wakeup thread pool-483-thread-3) 01:28:10.323 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:2118) Learned about tablet 8798d444603e4dd7943589ff83e108f8 for table 'testMultipleSessions' with partition [, ) 01:28:10.323 [DEBUG - New I/O worker #2153] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8798d444603e4dd7943589ff83e108f8@[1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409)[L],316c988cd2454ee19cb8a22f98a153e0(127.6.95.194:44821),5badfe537b704c53b985d5f918f31d1a(127.6.95.195:38961)]}] 01:28:10.324 [DEBUG - New I/O worker #2153] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=11), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.95.252:47527, [11ms] Sub RPC GetTableLocations: received response from server master-127.6.95.252:47527: OK, deferred=null) after lookup 01:28:10.324 [DEBUG - New I/O worker #2153] (Deferred.java:1330) callback=retry RPC@1455338154 returned Deferred@1809013173(state=PENDING, result=null, callback=(continuation of Deferred@111657793 after retry RPC@1455338154), errback=(continuation of Deferred@111657793 after retry RPC@1455338154)), so the following Deferred is getting paused: Deferred@111657793(state=PAUSED, result=Deferred@1809013173, callback=org.apache.kudu.client.AsyncKuduSession$2@def055b -> passthrough -> wakeup thread pool-483-thread-4, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2e67cdae -> wakeup thread pool-483-thread-4) 01:28:10.335 [DEBUG - New I/O worker #2155] (Negotiator.java:764) Authenticated connection [id: 0xfdc12d86, /127.6.95.193:49311 => /127.6.95.193:40409] using TOKEN/null 01:28:10.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:10.355221 7020 tablet.cc:1783] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:28:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.433445 7100 raft_consensus.cc:1181] T 8798d444603e4dd7943589ff83e108f8 P 5badfe537b704c53b985d5f918f31d1a [term 1 FOLLOWER]: Refusing update from remote peer 1f2026756db9406383d1481ef7a1a9c3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:28:10.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.437070 6969 raft_consensus.cc:1181] T 8798d444603e4dd7943589ff83e108f8 P 316c988cd2454ee19cb8a22f98a153e0 [term 1 FOLLOWER]: Refusing update from remote peer 1f2026756db9406383d1481ef7a1a9c3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:28:10.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.437700 7192 consensus_queue.cc:984] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5badfe537b704c53b985d5f918f31d1a" member_type: VOTER last_known_addr { host: "127.6.95.195" port: 38961 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:28:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.444357 7192 consensus_queue.cc:984] T 8798d444603e4dd7943589ff83e108f8 P 1f2026756db9406383d1481ef7a1a9c3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "316c988cd2454ee19cb8a22f98a153e0" member_type: VOTER last_known_addr { host: "127.6.95.194" port: 44821 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:28:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.477150 7258 mvcc.cc:203] Tried to move safe_time back from 6351128536717332480 to 6351128535804997632. Current Snapshot: MvccSnapshot[committed={T|T < 6351128536717332480}] 01:28:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:10.495507 7259 mvcc.cc:203] Tried to move safe_time back from 6351128536717332480 to 6351128535804997632. Current Snapshot: MvccSnapshot[committed={T|T < 6351128536717332480}] 01:28:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:11.180346 7257 mvcc.cc:203] Tried to move safe_time back from 6351128539811692544 to 6351128535804997632. Current Snapshot: MvccSnapshot[committed={T|T < 6351128539811692544 or (T in {6351128539811692544})}] 01:28:19.969 [DEBUG - New I/O worker #2152] (Connection.java:429) [peer master-127.6.95.253:51702(127.6.95.253:51702)] encountered a read timeout; closing the channel 01:28:19.969 [DEBUG - New I/O worker #2154] (Connection.java:429) [peer master-127.6.95.254:34354(127.6.95.254:34354)] encountered a read timeout; closing the channel 01:28:19.969 [DEBUG - New I/O worker #2154] (Connection.java:688) [peer master-127.6.95.254:34354(127.6.95.254:34354)] cleaning up while in state READY due to: [peer master-127.6.95.254:34354(127.6.95.254:34354)] encountered a read timeout; closing the channel 01:28:19.969 [DEBUG - New I/O worker #2152] (Connection.java:688) [peer master-127.6.95.253:51702(127.6.95.253:51702)] cleaning up while in state READY due to: [peer master-127.6.95.253:51702(127.6.95.253:51702)] encountered a read timeout; closing the channel 01:28:20.328 [DEBUG - New I/O worker #2153] (Connection.java:429) [peer master-127.6.95.252:47527(127.6.95.252:47527)] encountered a read timeout; closing the channel 01:28:20.328 [DEBUG - New I/O worker #2153] (Connection.java:688) [peer master-127.6.95.252:47527(127.6.95.252:47527)] cleaning up while in state READY due to: [peer master-127.6.95.252:47527(127.6.95.252:47527)] encountered a read timeout; closing the channel 01:28:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.223944 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1173, attempt_no=1}) took 1005ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":252,"num_ops":1,"prepare.queue_time_us":976445,"prepare.run_cpu_time_us":2027,"prepare.run_wall_time_us":11172,"replication_time_us":15263,"row_lock_wait_count":1,"row_lock_wait_us":9170,"spinlock_wait_cycles":168832}]]} 01:28:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.243144 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1174, attempt_no=1}) took 1010ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":288,"num_ops":1,"prepare.queue_time_us":973742,"prepare.run_cpu_time_us":2034,"prepare.run_wall_time_us":18709,"replication_time_us":15675,"row_lock_wait_count":1,"row_lock_wait_us":16732}]]} 01:28:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.266274 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1175, attempt_no=1}) took 1023ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":217,"num_ops":1,"prepare.queue_time_us":982484,"prepare.run_cpu_time_us":2027,"prepare.run_wall_time_us":17751,"replication_time_us":18591,"row_lock_wait_count":1,"row_lock_wait_us":15759,"spinlock_wait_cycles":104832}]]} 01:28:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.280972 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1176, attempt_no=1}) took 1026ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":265,"num_ops":1,"prepare.queue_time_us":988483,"prepare.run_cpu_time_us":2390,"prepare.run_wall_time_us":25939,"raft.queue_time_us":1154,"raft.run_cpu_time_us":554,"raft.run_wall_time_us":553,"replication_time_us":13618,"row_lock_wait_count":1,"row_lock_wait_us":19655,"spinlock_wait_cycles":1728256}]]} 01:28:24.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.306222 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1177, attempt_no=1}) took 1044ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":251,"num_ops":1,"prepare.queue_time_us":1007784,"prepare.run_cpu_time_us":2101,"prepare.run_wall_time_us":11611,"replication_time_us":16281,"row_lock_wait_count":1,"row_lock_wait_us":9553,"spinlock_wait_cycles":150400}]]} 01:28:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.318361 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1178, attempt_no=1}) took 1043ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":176,"num_ops":1,"prepare.queue_time_us":1005734,"prepare.run_cpu_time_us":2695,"prepare.run_wall_time_us":23434,"raft.queue_time_us":133,"raft.run_cpu_time_us":681,"raft.run_wall_time_us":681,"replication_time_us":12555,"row_lock_wait_count":1,"row_lock_wait_us":19386,"spinlock_wait_cycles":3631872}]]} 01:28:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.337867 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1179, attempt_no=1}) took 1046ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":369,"num_ops":1,"prepare.queue_time_us":1012715,"prepare.run_cpu_time_us":1690,"prepare.run_wall_time_us":11706,"replication_time_us":17743,"row_lock_wait_count":1,"row_lock_wait_us":10038,"spinlock_wait_cycles":1100544}]]} 01:28:24.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.354074 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1180, attempt_no=1}) took 1046ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":1106,"num_ops":1,"prepare.queue_time_us":1008326,"prepare.run_cpu_time_us":1819,"prepare.run_wall_time_us":19365,"replication_time_us":13989,"row_lock_wait_count":1,"row_lock_wait_us":17569,"spinlock_wait_cycles":2816}]]} 01:28:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.364734 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1181, attempt_no=1}) took 1039ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":119,"num_ops":1,"prepare.queue_time_us":1010679,"prepare.run_cpu_time_us":1595,"prepare.run_wall_time_us":16178,"replication_time_us":11220,"row_lock_wait_count":1,"row_lock_wait_us":14606}]]} 01:28:24.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.377562 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1182, attempt_no=1}) took 1034ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":147,"num_ops":1,"prepare.queue_time_us":1007660,"prepare.run_cpu_time_us":1639,"prepare.run_wall_time_us":12679,"replication_time_us":10639,"row_lock_wait_count":1,"row_lock_wait_us":11058,"spinlock_wait_cycles":198016}]]} 01:28:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:28:24.389622 7326 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=1183, attempt_no=1}) took 1031ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":106,"num_ops":1,"prepare.queue_time_us":1005806,"prepare.run_cpu_time_us":1468,"prepare.run_wall_time_us":12584,"replication_time_us":8865,"row_lock_wait_count":1,"row_lock_wait_us":11134}]]} 01:28:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:30.460116 6773 debug-util.cc:397] Leaking SignalData structure 0x7b0800029080 after lost signal to thread 6739 01:28:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:30.463820 6773 debug-util.cc:397] Leaking SignalData structure 0x7b08000a0380 after lost signal to thread 6879 01:28:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:39.422900 6773 debug-util.cc:397] Leaking SignalData structure 0x7b08000a0200 after lost signal to thread 6739 01:28:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:39.424863 6773 debug-util.cc:397] Leaking SignalData structure 0x7b08000292c0 after lost signal to thread 6879 01:28:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:50.778292 6773 debug-util.cc:397] Leaking SignalData structure 0x7b08000a01e0 after lost signal to thread 6739 01:28:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:50.780376 6773 debug-util.cc:397] Leaking SignalData structure 0x7b08000a0360 after lost signal to thread 6879 01:28:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:53.973145 7035 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 7023 01:28:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:28:53.974177 7035 debug-util.cc:397] Leaking SignalData structure 0x7b08000bc300 after lost signal to thread 7141 01:29:09.928 [DEBUG - Time-limited test] (Deferred.java:1189) While joining Deferred@1493187182(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$2@2948c3c5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11ce2f9d -> wakeup thread Time-limited test): interrupted 01:29:09.932 [DEBUG - Test worker] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:29:09.932 [DEBUG - New I/O worker #2155] (Connection.java:688) [peer 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409)] cleaning up while in state READY due to: connection disconnected 01:29:09.932 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.933 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.933 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.933 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.933 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.934 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.934 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.935 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.935 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.935 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.935 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.937 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.937 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.938 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.939 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.940 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.940 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.942 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.942 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.942 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.944 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.944 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.944 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.946 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.946 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.946 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.948 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:09.946314 6753 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.6.95.193:49311 (ReqId={client: f2d071a78c8a496e8d9451356febbce6, seq_no=5274, attempt_no=1}) could send its response 01:29:09.948 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.949 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.950 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.950 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.951 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.954 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:09.952284 6527 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 6726 01:29:09.954 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.954 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.955 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.955 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.955 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.955 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.955 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.956 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.956 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.956 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.956 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.957 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.957 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.957 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.958 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.958 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.958 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.959 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.959 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.959 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.959 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.960 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.960 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.960 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.960 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.960 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.961 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.961 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.961 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.962 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.962 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.962 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.963 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.963 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.963 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.964 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.964 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.964 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.965 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.965 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.965 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.965 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.966 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.966 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.966 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.967 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.967 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.967 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.968 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.968 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.968 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.969 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.969 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.969 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.970 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.970 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.970 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.970 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.971 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.971 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.971 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.972 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.972 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.972 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.973 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.973 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.973 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.974 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.977 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.977 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.977 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.978 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.978 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.978 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.978 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.978 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.979 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.979 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.979 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.979 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.980 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.980 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.980 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.981 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.981 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.981 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.982 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.982 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.982 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.982 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.983 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.983 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.983 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.983 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.984 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.984 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.984 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.984 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.985 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.985 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.986 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.986 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.987 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.987 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.988 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.988 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.988 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.988 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.989 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.989 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.989 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.990 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.990 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.990 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.990 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.990 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.991 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.991 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.991 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.992 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.992 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.992 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.993 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.993 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.993 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.994 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.994 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.994 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.995 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.995 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.995 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.995 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.996 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.996 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.996 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.996 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.997 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.997 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.997 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.997 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.997 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.997 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.998 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.998 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.998 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.998 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:09.998 [INFO - New I/O worker #2155] (AsyncKuduClient.java:1970) Invalidating location 1f2026756db9406383d1481ef7a1a9c3(127.6.95.193:40409) for tablet 8798d444603e4dd7943589ff83e108f8: connection disconnected 01:29:09.998 [DEBUG - New I/O worker #2155] (RemoteTablet.java:131) tablet 8798d444603e4dd7943589ff83e108f8 already removed ts 1f2026756db9406383d1481ef7a1a9c3, size left is 2 01:29:09.998 [WARN - New I/O worker #2155] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:29:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.024515 6608 connection.cc:647] server connection from 127.6.95.193:34351 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.193:34351): Connection reset by peer (error 104) 01:29:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.028242 7032 connection.cc:647] server connection from 127.6.95.193:35957 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:29:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.029215 7032 connection.cc:320] Error closing socket: Network error: TlsSocket::Close: Broken pipe 01:29:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.028997 6674 connection.cc:647] server connection from 127.6.95.193:34386 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.193:34386): Connection reset by peer (error 104) 01:29:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.028997 6544 connection.cc:647] server connection from 127.6.95.193:55087 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.193:55087): Connection reset by peer (error 104) 01:29:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.030522 6901 connection.cc:647] server connection from 127.6.95.193:33764 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:29:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.032107 6901 connection.cc:320] Error closing socket: Network error: TlsSocket::Close: Broken pipe 01:29:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:10.032315 6527 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 6882 01:29:10.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:10.065893 6527 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 7016 01:29:10.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.066185 6673 connection.cc:647] server connection from 127.6.95.194:54523 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.194:54523): Connection reset by peer (error 104) 01:29:10.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.066557 6543 connection.cc:647] server connection from 127.6.95.194:36304 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.194:36304): Connection reset by peer (error 104) 01:29:10.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.069025 6607 connection.cc:647] server connection from 127.6.95.194:42482 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.194:42482): Connection reset by peer (error 104) 01:29:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.103744 6545 connection.cc:647] server connection from 127.6.95.195:35830 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.195:35830): Connection reset by peer (error 104) 01:29:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.103842 6607 connection.cc:647] server connection from 127.6.95.195:45762 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.195:45762): Connection reset by peer (error 104) 01:29:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:10.104037 6673 connection.cc:647] server connection from 127.6.95.195:47396 recv error: Network error: failed to read from TLS socket (remote: 127.6.95.195:47396): Connection reset by peer (error 104) 01:29:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:10.105695 6527 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 6535 01:29:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:10.128499 6527 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 6597 01:29:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:10.149158 6527 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 6662 01:29:11.204 [ERROR - Test worker] (RetryRule.java:80) testMultipleSessions(org.apache.kudu.client.ITClientStress): failed attempt 2 org.junit.runners.model.TestTimedOutException: test timed out after 60000 milliseconds at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.stumbleupon.async.Deferred.doJoin(Deferred.java:1136) at com.stumbleupon.async.Deferred.join(Deferred.java:1019) at org.apache.kudu.client.KuduSession.apply(KuduSession.java:81) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:163) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:150) at org.apache.kudu.client.ITClientStress$1.run(ITClientStress.java:80) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:2022) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at org.apache.kudu.client.ITClientStress.runTasks(ITClientStress.java:86) at org.apache.kudu.client.ITClientStress.testMultipleSessions(ITClientStress.java:147) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 01:29:11.206 [INFO - Test worker] (KuduTestHarness.java:140) Creating a new MiniKuduCluster... 01:29:11.206 [INFO - Test worker] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 01:29:11.206 [INFO - Test worker] (KuduBinaryLocator.java:78) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 01:29:11.207 [INFO - Test worker] (MiniKuduCluster.java:187) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 01:29:11.209 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238" hms_mode: NONE mini_kdc_options { } } 01:29:11.480 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: 01:29:11.480 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: start_cluster { } 01:29:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.513186 7563 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:29:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:29:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/wal 01:29:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data 01:29:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:29:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.254:35261 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.254 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/logs 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data/info.pb 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:29:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 with env {} 01:29:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:29:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:11.735466 7575 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:29:11.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:11.735970 7575 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:29:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:11.747661 7575 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:29:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:11.747833 7575 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:29:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:11.748067 7575 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:29:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:11.748288 7575 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:29:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.774511 7575 master_main.cc:73] Master server non-default flags: 01:29:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data 01:29:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/wal 01:29:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:29:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.254:35261 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data/info.pb 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.254 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/logs 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:29:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:29:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.792178 7575 master_main.cc:80] Initializing master server... 01:29:11.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.794878 7575 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 155378us 01:29:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.795915 7575 fs_manager.cc:263] Metadata directory not provided 01:29:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.796066 7575 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:29:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.796561 7575 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:29:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.796700 7575 server_base.cc:434] Attempting to create new FS layout instead 01:29:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.801785 7575 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data/instance: 01:29:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "04ef1e6ea730487aaca49ecd2431521b" 01:29:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:11 on quasar-aecznm-1.vpc.cloudera.com" 01:29:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.802850 7575 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/wal/instance: 01:29:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "04ef1e6ea730487aaca49ecd2431521b" 01:29:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:11 on quasar-aecznm-1.vpc.cloudera.com" 01:29:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.821790 7575 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.008s sys 0.008s 01:29:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.823495 7575 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:29:11.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.825232 7575 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:29:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.834846 7575 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.835001 7575 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/wal 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "04ef1e6ea730487aaca49ecd2431521b" 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:11 on quasar-aecznm-1.vpc.cloudera.com" 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:11.835310 7575 fs_report.cc:352] FS layout report 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/wal 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/wal 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data/data 01:29:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:29:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:29:12.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.003235 7575 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:29:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.006400 7575 master_main.cc:83] Starting Master server... 01:29:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.135627 7575 rpc_server.cc:205] RPC server started. Bound to: 127.7.98.254:35261 01:29:12.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.137013 7575 webserver.cc:175] Starting webserver on 127.7.98.254:0 01:29:12.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.137161 7575 webserver.cc:186] Document root disabled 01:29:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.140017 7575 webserver.cc:313] Webserver started. Bound to: http://127.7.98.254:53536/ 01:29:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.141196 7575 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-0/data/info.pb 01:29:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.144207 7563 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 7575 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.145320 7563 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/wal 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.253:57750 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.253 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/logs 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data/info.pb 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 with env {} 01:29:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.145911 7636 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:29:12.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.162351 7636 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } has no permanent_uuid. Determining permanent_uuid... 01:29:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.188966 7636 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } has no permanent_uuid. Determining permanent_uuid... 01:29:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.191365 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.253:57750: Network error: Client connection negotiation failed: client connection to 127.7.98.253:57750: connect: Connection refused (error 111) 01:29:12.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.210939 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } attempt: 1 01:29:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.212566 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.253:57750: Network error: Client connection negotiation failed: client connection to 127.7.98.253:57750: connect: Connection refused (error 111) 01:29:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.260951 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } attempt: 2 01:29:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.262630 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.253:57750: Network error: Client connection negotiation failed: client connection to 127.7.98.253:57750: connect: Connection refused (error 111) 01:29:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:29:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.373703 7637 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:29:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.374084 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } attempt: 3 01:29:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.374292 7637 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:29:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.375543 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.253:57750: Network error: Client connection negotiation failed: client connection to 127.7.98.253:57750: connect: Connection refused (error 111) 01:29:12.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.386509 7637 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:29:12.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.386700 7637 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:29:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.387030 7637 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:29:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.387219 7637 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:29:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.412269 7637 master_main.cc:73] Master server non-default flags: 01:29:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data 01:29:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/wal 01:29:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:29:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.253:57750 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data/info.pb 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.253 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/logs 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:29:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:29:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:29:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:29:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:29:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:29:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:29:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:29:12.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.429401 7637 master_main.cc:80] Initializing master server... 01:29:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.432518 7637 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 155878us 01:29:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.433583 7637 fs_manager.cc:263] Metadata directory not provided 01:29:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.433771 7637 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:29:12.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.434207 7637 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:29:12.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.434365 7637 server_base.cc:434] Attempting to create new FS layout instead 01:29:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.439445 7637 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data/instance: 01:29:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "4e4f0e6287fd4d8eaee18027482adb6a" 01:29:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:12 on quasar-aecznm-1.vpc.cloudera.com" 01:29:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.440456 7637 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/wal/instance: 01:29:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "4e4f0e6287fd4d8eaee18027482adb6a" 01:29:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:12 on quasar-aecznm-1.vpc.cloudera.com" 01:29:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.458619 7637 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.007s sys 0.008s 01:29:12.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.460306 7637 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:29:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.462097 7637 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:29:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.472004 7637 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s 01:29:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.472163 7637 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/wal 01:29:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "4e4f0e6287fd4d8eaee18027482adb6a" 01:29:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:12 on quasar-aecznm-1.vpc.cloudera.com" 01:29:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.472493 7637 fs_report.cc:352] FS layout report 01:29:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/wal 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/wal 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data/data 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:29:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:29:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.551955 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } attempt: 4 01:29:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.553890 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.253:57750: Network error: Client connection negotiation failed: client connection to 127.7.98.253:57750: connect: Connection refused (error 111) 01:29:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.747117 7637 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:29:12.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.750907 7637 master_main.cc:83] Starting Master server... 01:29:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.857296 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } attempt: 5 01:29:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.859009 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.253:57750: Network error: Client connection negotiation failed: client connection to 127.7.98.253:57750: connect: Connection refused (error 111) 01:29:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.874603 7637 rpc_server.cc:205] RPC server started. Bound to: 127.7.98.253:57750 01:29:12.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.875607 7637 webserver.cc:175] Starting webserver on 127.7.98.253:0 01:29:12.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.875730 7637 webserver.cc:186] Document root disabled 01:29:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.878171 7637 webserver.cc:313] Webserver started. Bound to: http://127.7.98.253:57164/ 01:29:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.879550 7637 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-1/data/info.pb 01:29:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.880993 7563 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 7637 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.882223 7563 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/wal 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:29:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.252:36904 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.252 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/logs 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data/info.pb 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:29:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 with env {} 01:29:12.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.885361 7700 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:29:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.902874 7700 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } has no permanent_uuid. Determining permanent_uuid... 01:29:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.932315 7700 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } has no permanent_uuid. Determining permanent_uuid... 01:29:12.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.947161 7700 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } has no permanent_uuid. Determining permanent_uuid... 01:29:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.948953 7700 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:12.968444 7700 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 1 01:29:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:12.970266 7700 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.018651 7700 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 2 01:29:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.020303 7700 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:29:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.106701 7701 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:29:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.107152 7701 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:29:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.119140 7701 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:29:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.119289 7701 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:29:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.119395 7701 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:29:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.119571 7701 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:29:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.131703 7700 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 3 01:29:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.133389 7700 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.145153 7701 master_main.cc:73] Master server non-default flags: 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/wal 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.252:36904 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data/info.pb 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.252 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/logs 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:29:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:29:13.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.161734 7701 master_main.cc:80] Initializing master server... 01:29:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.164302 7701 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 156378us 01:29:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.165302 7701 fs_manager.cc:263] Metadata directory not provided 01:29:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.165463 7701 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:29:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.165818 7701 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:29:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.165961 7701 server_base.cc:434] Attempting to create new FS layout instead 01:29:13.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.171075 7701 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data/instance: 01:29:13.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" 01:29:13.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:13 on quasar-aecznm-1.vpc.cloudera.com" 01:29:13.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.172062 7701 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/wal/instance: 01:29:13.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" 01:29:13.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:13 on quasar-aecznm-1.vpc.cloudera.com" 01:29:13.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.189863 7701 fs_manager.cc:503] Time spent creating directory manager: real 0.017s user 0.008s sys 0.008s 01:29:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.191349 7701 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:29:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.193171 7701 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:29:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.203166 7701 fs_manager.cc:419] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s 01:29:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.203312 7701 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/wal 01:29:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" 01:29:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:13 on quasar-aecznm-1.vpc.cloudera.com" 01:29:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.203577 7701 fs_report.cc:352] FS layout report 01:29:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/wal 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/wal 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data/data 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:29:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:29:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.309749 7700 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 4 01:29:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.311836 7700 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.393453 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } attempt: 6 01:29:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.403697 7636 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } has no permanent_uuid. Determining permanent_uuid... 01:29:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.405311 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.437585 7701 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:29:13.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.437710 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 1 01:29:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.440044 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.440996 7701 master_main.cc:83] Starting Master server... 01:29:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.508410 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 2 01:29:13.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.511234 7636 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.7.98.252:36904: Network error: Client connection negotiation failed: client connection to 127.7.98.252:36904: connect: Connection refused (error 111) 01:29:13.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.566150 7701 rpc_server.cc:205] RPC server started. Bound to: 127.7.98.252:36904 01:29:13.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.567270 7701 webserver.cc:175] Starting webserver on 127.7.98.252:0 01:29:13.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.567423 7701 webserver.cc:186] Document root disabled 01:29:13.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.569866 7701 webserver.cc:313] Webserver started. Bound to: http://127.7.98.252:48623/ 01:29:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.570896 7701 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/master-2/data/info.pb 01:29:13.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.575184 7766 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:29:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.577365 7563 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 7701 01:29:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.579061 7563 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:29:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/wal 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.193:0 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.7.98.193 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.193 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/logs 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data/info.pb 01:29:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:29:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:29:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:29:13.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.587625 7636 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 3 01:29:13.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.603667 7766 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } has no permanent_uuid. Determining permanent_uuid... 01:29:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.615221 7700 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } attempt: 5 01:29:13.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.632072 7636 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: Bootstrap starting. 01:29:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.636283 7636 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: No blocks or log segments found. Creating new log. 01:29:13.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.638346 7636 log.cc:525] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: Log is configured to *not* fsync() on all Append() calls 01:29:13.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.642963 7766 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } has no permanent_uuid. Determining permanent_uuid... 01:29:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.648216 7700 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: Bootstrap starting. 01:29:13.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.652995 7700 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: No blocks or log segments found. Creating new log. 01:29:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.655038 7636 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: No bootstrap required, opened a new log 01:29:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.655628 7766 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } has no permanent_uuid. Determining permanent_uuid... 01:29:13.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.665388 7700 log.cc:525] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: Log is configured to *not* fsync() on all Append() calls 01:29:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.676558 7766 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: Bootstrap starting. 01:29:13.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.679021 7636 raft_consensus.cc:340] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.681962 7700 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: No bootstrap required, opened a new log 01:29:13.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.683151 7766 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: No blocks or log segments found. Creating new log. 01:29:13.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.686014 7766 log.cc:525] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: Log is configured to *not* fsync() on all Append() calls 01:29:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.680219 7636 raft_consensus.cc:366] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:29:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.691491 7636 raft_consensus.cc:705] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04ef1e6ea730487aaca49ecd2431521b, State: Initialized, Role: FOLLOWER 01:29:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.692472 7636 consensus_queue.cc:229] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.693953 7766 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: No bootstrap required, opened a new log 01:29:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.697923 7774 sys_catalog.cc:337] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:13.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.699090 7774 sys_catalog.cc:340] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: This master's current role is: FOLLOWER 01:29:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.702994 7636 sys_catalog.cc:337] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.703713 7636 sys_catalog.cc:340] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: This master's current role is: FOLLOWER 01:29:13.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.705191 7636 sys_catalog.cc:424] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: configured and running, proceeding with master startup. 01:29:13.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.711854 7575 master_main.cc:86] Master server successfully started. 01:29:13.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.716236 7700 raft_consensus.cc:340] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.717293 7700 raft_consensus.cc:366] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:29:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.717811 7700 raft_consensus.cc:705] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e4f0e6287fd4d8eaee18027482adb6a, State: Initialized, Role: FOLLOWER 01:29:13.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.718947 7700 consensus_queue.cc:229] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.720410 7766 raft_consensus.cc:340] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.721499 7766 raft_consensus.cc:366] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:29:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.722043 7766 raft_consensus.cc:705] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b03f0fd2ab524a5db8fd7c84562dac3b, State: Initialized, Role: FOLLOWER 01:29:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.724043 7766 consensus_queue.cc:229] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.728368 7700 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.729125 7700 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: This master's current role is: FOLLOWER 01:29:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.730599 7700 sys_catalog.cc:424] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: configured and running, proceeding with master startup. 01:29:13.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.731617 7778 sys_catalog.cc:337] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.755614 7778 sys_catalog.cc:340] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: This master's current role is: FOLLOWER 01:29:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.722930 7775 catalog_manager.cc:1104] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:29:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.773156 7775 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:29:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.732487 7766 sys_catalog.cc:337] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.879601 7766 sys_catalog.cc:340] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: This master's current role is: FOLLOWER 01:29:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.881469 7766 sys_catalog.cc:424] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: configured and running, proceeding with master startup. 01:29:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.762123 7592 tablet.cc:1783] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:29:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:29:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.881798 7767 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:29:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.884909 7767 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:29:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.901290 7701 master_main.cc:86] Master server successfully started. 01:29:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.907294 7767 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:29:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.908161 7767 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.98.193 01:29:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.911695 7780 catalog_manager.cc:1104] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:29:13.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.912025 7780 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:29:13.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.945519 7722 tablet.cc:1783] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:29:13.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.732524 7777 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:13.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.951910 7777 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: This master's current role is: FOLLOWER 01:29:13.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.953863 7637 master_main.cc:86] Master server successfully started. 01:29:13.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.968381 7777 raft_consensus.cc:472] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:29:13.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.968294 7767 tablet_server_main.cc:78] Tablet server non-default flags: 01:29:13.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/wal 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.193:0 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data/info.pb 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.193 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.7.98.193 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/logs 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:29:13.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:29:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:29:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:29:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:29:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.968933 7777 raft_consensus.cc:494] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.972230 7777 leader_election.cc:296] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 04ef1e6ea730487aaca49ecd2431521b (127.7.98.254:35261), b03f0fd2ab524a5db8fd7c84562dac3b (127.7.98.252:36904) 01:29:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.973973 7613 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04ef1e6ea730487aaca49ecd2431521b" is_pre_election: true 01:29:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.974511 7743 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" is_pre_election: true 01:29:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.974912 7613 raft_consensus.cc:2346] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e4f0e6287fd4d8eaee18027482adb6a in term 0. 01:29:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.975502 7743 raft_consensus.cc:2346] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e4f0e6287fd4d8eaee18027482adb6a in term 0. 01:29:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.976234 7649 leader_election.cc:310] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef1e6ea730487aaca49ecd2431521b, 4e4f0e6287fd4d8eaee18027482adb6a; no voters: 01:29:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.977212 7777 raft_consensus.cc:2640] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Leader pre-election won for term 1 01:29:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.977501 7777 raft_consensus.cc:472] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:29:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.977829 7777 raft_consensus.cc:2886] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 0 FOLLOWER]: Advancing to term 1 01:29:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.988219 7777 raft_consensus.cc:494] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:13.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.992811 7777 leader_election.cc:296] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [CANDIDATE]: Term 1 election: Requested vote from peers 04ef1e6ea730487aaca49ecd2431521b (127.7.98.254:35261), b03f0fd2ab524a5db8fd7c84562dac3b (127.7.98.252:36904) 01:29:13.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.993945 7613 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04ef1e6ea730487aaca49ecd2431521b" 01:29:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.994345 7743 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" 01:29:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.994539 7613 raft_consensus.cc:2886] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 0 FOLLOWER]: Advancing to term 1 01:29:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:13.994922 7743 raft_consensus.cc:2886] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 0 FOLLOWER]: Advancing to term 1 01:29:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.001569 7613 raft_consensus.cc:2346] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e4f0e6287fd4d8eaee18027482adb6a in term 1. 01:29:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.002466 7743 raft_consensus.cc:2346] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e4f0e6287fd4d8eaee18027482adb6a in term 1. 01:29:14.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.003917 7649 leader_election.cc:310] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef1e6ea730487aaca49ecd2431521b, 4e4f0e6287fd4d8eaee18027482adb6a; no voters: 01:29:14.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.004863 7777 raft_consensus.cc:2640] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 1 FOLLOWER]: Leader election won for term 1 01:29:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:13.756322 7656 tablet.cc:1783] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:29:14.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.011943 7767 tablet_server_main.cc:85] Initializing tablet server... 01:29:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.017241 7767 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 156878us 01:29:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.018287 7767 fs_manager.cc:263] Metadata directory not provided 01:29:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.018676 7767 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:29:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.019352 7767 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:29:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.019672 7767 server_base.cc:434] Attempting to create new FS layout instead 01:29:14.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.021044 7777 raft_consensus.cc:667] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [term 1 LEADER]: Becoming Leader. State: Replica: 4e4f0e6287fd4d8eaee18027482adb6a, State: Running, Role: LEADER 01:29:14.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.027164 7777 consensus_queue.cc:206] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } 01:29:14.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.027482 7767 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data/instance: 01:29:14.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "a1170895397248e39a195e2b9e398f75" 01:29:14.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:14 on quasar-aecznm-1.vpc.cloudera.com" 01:29:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.028983 7767 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/wal/instance: 01:29:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "a1170895397248e39a195e2b9e398f75" 01:29:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:14 on quasar-aecznm-1.vpc.cloudera.com" 01:29:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.054944 7786 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4e4f0e6287fd4d8eaee18027482adb6a. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.056195 7786 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: This master's current role is: LEADER 01:29:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.065062 7767 fs_manager.cc:503] Time spent creating directory manager: real 0.035s user 0.016s sys 0.006s 01:29:14.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.067682 7767 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:29:14.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.070528 7767 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:29:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:14.089718 7779 catalog_manager.cc:1104] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:29:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:14.090044 7779 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:29:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.090417 7791 catalog_manager.cc:1046] Loading table and tablet metadata into memory... 01:29:14.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.093706 7767 fs_manager.cc:419] Time spent opening block manager: real 0.009s user 0.001s sys 0.005s 01:29:14.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.093968 7767 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/wal 01:29:14.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "a1170895397248e39a195e2b9e398f75" 01:29:14.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:14 on quasar-aecznm-1.vpc.cloudera.com" 01:29:14.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.094563 7791 catalog_manager.cc:1057] Initializing Kudu internal certificate authority... 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.094444 7767 fs_report.cc:352] FS layout report 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/wal 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/wal 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data/data 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:29:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:29:14.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.169093 7743 raft_consensus.cc:1181] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [term 1 FOLLOWER]: Refusing update from remote peer 4e4f0e6287fd4d8eaee18027482adb6a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:29:14.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.170483 7613 raft_consensus.cc:1181] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [term 1 FOLLOWER]: Refusing update from remote peer 4e4f0e6287fd4d8eaee18027482adb6a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:29:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.172968 7786 consensus_queue.cc:984] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [LEADER]: Connected to new peer: Peer: permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:29:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.173632 7777 consensus_queue.cc:984] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [LEADER]: Connected to new peer: Peer: permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:29:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.201473 7778 sys_catalog.cc:337] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4e4f0e6287fd4d8eaee18027482adb6a. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.202317 7778 sys_catalog.cc:340] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: This master's current role is: FOLLOWER 01:29:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.215734 7774 sys_catalog.cc:337] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4e4f0e6287fd4d8eaee18027482adb6a. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.216586 7774 sys_catalog.cc:340] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: This master's current role is: FOLLOWER 01:29:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.216962 7797 mvcc.cc:203] Tried to move safe_time back from 6351128797851463680 to 6351128797307015168. Current Snapshot: MvccSnapshot[committed={T|T < 6351128797851463680}] 01:29:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.219595 7777 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.220141 7798 mvcc.cc:203] Tried to move safe_time back from 6351128797851463680 to 6351128797307015168. Current Snapshot: MvccSnapshot[committed={T|T < 6351128797851463680}] 01:29:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.220401 7777 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: This master's current role is: LEADER 01:29:14.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.220971 7777 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.221639 7777 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a [sys.catalog]: This master's current role is: LEADER 01:29:14.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.222326 7791 catalog_manager.cc:941] Generated new certificate authority record 01:29:14.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.225641 7791 catalog_manager.cc:1066] Loading token signing keys... 01:29:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.228534 7774 sys_catalog.cc:337] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.229367 7774 sys_catalog.cc:340] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b [sys.catalog]: This master's current role is: FOLLOWER 01:29:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.232663 7778 sys_catalog.cc:337] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef1e6ea730487aaca49ecd2431521b" member_type: VOTER last_known_addr { host: "127.7.98.254" port: 35261 } } peers { permanent_uuid: "4e4f0e6287fd4d8eaee18027482adb6a" member_type: VOTER last_known_addr { host: "127.7.98.253" port: 57750 } } peers { permanent_uuid: "b03f0fd2ab524a5db8fd7c84562dac3b" member_type: VOTER last_known_addr { host: "127.7.98.252" port: 36904 } } } 01:29:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.233629 7778 sys_catalog.cc:340] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b [sys.catalog]: This master's current role is: FOLLOWER 01:29:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.261116 7791 catalog_manager.cc:4095] T 00000000000000000000000000000000 P 4e4f0e6287fd4d8eaee18027482adb6a: Generated new TSK 0 01:29:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.464610 7767 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:29:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.475153 7767 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:29:14.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.486502 7767 tablet_server_main.cc:90] Starting tablet server... 01:29:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.781714 7775 catalog_manager.cc:1102] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: acquiring CA information for follower catalog manager: success 01:29:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.794064 7775 catalog_manager.cc:1130] T 00000000000000000000000000000000 P 04ef1e6ea730487aaca49ecd2431521b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 01:29:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.920451 7780 catalog_manager.cc:1102] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: acquiring CA information for follower catalog manager: success 01:29:14.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:14.924567 7780 catalog_manager.cc:1130] T 00000000000000000000000000000000 P b03f0fd2ab524a5db8fd7c84562dac3b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 01:29:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.259706 7767 rpc_server.cc:205] RPC server started. Bound to: 127.7.98.193:54247 01:29:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.261656 7767 webserver.cc:175] Starting webserver on 127.7.98.193:0 01:29:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.262105 7767 webserver.cc:186] Document root disabled 01:29:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.277088 7767 webserver.cc:313] Webserver started. Bound to: http://127.7.98.193:49331/ 01:29:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.278561 7767 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-0/data/info.pb 01:29:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.290324 7563 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 7767 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.292045 7563 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/wal 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.194:0 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.7.98.194 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.194 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/logs 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data/info.pb 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:29:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:29:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.375721 7767 tablet_server_main.cc:93] Tablet server successfully started. 01:29:15.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.390030 7917 heartbeater.cc:345] Connected to a master server at 127.7.98.253:57750 01:29:15.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.390528 7917 heartbeater.cc:426] Registering TS with master... 01:29:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.391611 7917 heartbeater.cc:475] Master 127.7.98.253:57750 requested a full tablet report, sending... 01:29:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.395789 7667 ts_manager.cc:98] Registered new tserver with Master: a1170895397248e39a195e2b9e398f75 (127.7.98.193:54247) 01:29:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.399420 7914 heartbeater.cc:345] Connected to a master server at 127.7.98.252:36904 01:29:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.399713 7914 heartbeater.cc:426] Registering TS with master... 01:29:15.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.400503 7667 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.7.98.193:54683 01:29:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.400962 7914 heartbeater.cc:475] Master 127.7.98.252:36904 requested a full tablet report, sending... 01:29:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.405432 7733 ts_manager.cc:98] Registered new tserver with Master: a1170895397248e39a195e2b9e398f75 (127.7.98.193:54247) 01:29:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.392648 7919 heartbeater.cc:345] Connected to a master server at 127.7.98.254:35261 01:29:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.425621 7919 heartbeater.cc:426] Registering TS with master... 01:29:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.426146 7919 heartbeater.cc:475] Master 127.7.98.254:35261 requested a full tablet report, sending... 01:29:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.429937 7603 ts_manager.cc:98] Registered new tserver with Master: a1170895397248e39a195e2b9e398f75 (127.7.98.193:54247) 01:29:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:29:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:15.656796 7918 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:29:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:15.658007 7918 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:29:15.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:15.677084 7918 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:29:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:15.677933 7918 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.98.194 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.730952 7918 tablet_server_main.cc:78] Tablet server non-default flags: 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/wal 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.194:0 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data/info.pb 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.194 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.7.98.194 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/logs 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:29:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:29:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.761982 7918 tablet_server_main.cc:85] Initializing tablet server... 01:29:15.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.765986 7918 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 157378us 01:29:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.767026 7918 fs_manager.cc:263] Metadata directory not provided 01:29:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.767416 7918 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:29:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.768061 7918 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:29:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.768450 7918 server_base.cc:434] Attempting to create new FS layout instead 01:29:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.774871 7918 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data/instance: 01:29:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "a82b579bdd3d49869b0e45fd42da09f5" 01:29:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:15 on quasar-aecznm-1.vpc.cloudera.com" 01:29:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.776571 7918 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/wal/instance: 01:29:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "a82b579bdd3d49869b0e45fd42da09f5" 01:29:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:15 on quasar-aecznm-1.vpc.cloudera.com" 01:29:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.807513 7918 fs_manager.cc:503] Time spent creating directory manager: real 0.030s user 0.013s sys 0.013s 01:29:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.810334 7918 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:29:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.811568 7918 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:29:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.832000 7918 fs_manager.cc:419] Time spent opening block manager: real 0.007s user 0.003s sys 0.003s 01:29:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.832481 7918 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/wal 01:29:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "a82b579bdd3d49869b0e45fd42da09f5" 01:29:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:15 on quasar-aecznm-1.vpc.cloudera.com" 01:29:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:15.833200 7918 fs_report.cc:352] FS layout report 01:29:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:29:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/wal 01:29:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/wal 01:29:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data/data 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:29:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:29:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.257159 7918 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:29:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.266181 7918 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:29:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.272670 7918 tablet_server_main.cc:90] Starting tablet server... 01:29:16.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.405830 7917 heartbeater.cc:467] Master 127.7.98.253:57750 was elected leader, sending a full tablet report... 01:29:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.662005 7918 rpc_server.cc:205] RPC server started. Bound to: 127.7.98.194:50881 01:29:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.663410 7918 webserver.cc:175] Starting webserver on 127.7.98.194:0 01:29:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.663564 7918 webserver.cc:186] Document root disabled 01:29:16.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.665771 7918 webserver.cc:313] Webserver started. Bound to: http://127.7.98.194:34953/ 01:29:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.666723 7918 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-1/data/info.pb 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.670274 7563 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 7918 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.671645 7563 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/wal 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.195:0 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.7.98.195 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.195 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/logs 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data/info.pb 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:29:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:29:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:29:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.710438 7918 tablet_server_main.cc:93] Tablet server successfully started. 01:29:16.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.751933 8050 heartbeater.cc:345] Connected to a master server at 127.7.98.253:57750 01:29:16.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.752353 8050 heartbeater.cc:426] Registering TS with master... 01:29:16.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.753146 8050 heartbeater.cc:475] Master 127.7.98.253:57750 requested a full tablet report, sending... 01:29:16.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.755986 7667 ts_manager.cc:98] Registered new tserver with Master: a82b579bdd3d49869b0e45fd42da09f5 (127.7.98.194:50881) 01:29:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.758095 7667 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.7.98.194:39109 01:29:16.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.768879 8049 heartbeater.cc:345] Connected to a master server at 127.7.98.252:36904 01:29:16.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.769176 8049 heartbeater.cc:426] Registering TS with master... 01:29:16.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.769635 8049 heartbeater.cc:475] Master 127.7.98.252:36904 requested a full tablet report, sending... 01:29:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.771430 8051 heartbeater.cc:345] Connected to a master server at 127.7.98.254:35261 01:29:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.771709 8051 heartbeater.cc:426] Registering TS with master... 01:29:16.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.772162 8051 heartbeater.cc:475] Master 127.7.98.254:35261 requested a full tablet report, sending... 01:29:16.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.772850 7733 ts_manager.cc:98] Registered new tserver with Master: a82b579bdd3d49869b0e45fd42da09f5 (127.7.98.194:50881) 01:29:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.775660 7603 ts_manager.cc:98] Registered new tserver with Master: a82b579bdd3d49869b0e45fd42da09f5 (127.7.98.194:50881) 01:29:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:29:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:16.867516 8054 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:29:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:16.868232 8054 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:29:16.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:16.878895 8054 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:29:16.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:16.879544 8054 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.98.195 01:29:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.901265 8054 tablet_server_main.cc:78] Tablet server non-default flags: 01:29:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data 01:29:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/wal 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.7.98.195:0 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data/info.pb 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.7.98.195 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.7.98.254:35261,127.7.98.253:57750,127.7.98.252:36904 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.7.98.195 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/logs 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:29:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:29:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:29:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:29:16.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.920667 8054 tablet_server_main.cc:85] Initializing tablet server... 01:29:16.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.923545 8054 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 157878us 01:29:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.924302 8054 fs_manager.cc:263] Metadata directory not provided 01:29:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.924480 8054 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:29:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.925038 8054 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:29:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.925200 8054 server_base.cc:434] Attempting to create new FS layout instead 01:29:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.930179 8054 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data/instance: 01:29:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "c0d03701d5354bceb19d248d017859bf" 01:29:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:16 on quasar-aecznm-1.vpc.cloudera.com" 01:29:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.931243 8054 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/wal/instance: 01:29:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "c0d03701d5354bceb19d248d017859bf" 01:29:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:16 on quasar-aecznm-1.vpc.cloudera.com" 01:29:16.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.946166 8054 fs_manager.cc:503] Time spent creating directory manager: real 0.014s user 0.010s sys 0.003s 01:29:16.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.947898 8054 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:29:16.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.948549 8054 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:29:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.957796 8054 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s 01:29:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.958061 8054 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/wal 01:29:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "c0d03701d5354bceb19d248d017859bf" 01:29:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:29:16 on quasar-aecznm-1.vpc.cloudera.com" 01:29:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:16.958431 8054 fs_report.cc:352] FS layout report 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/wal 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/wal 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data/data 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:29:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:29:17.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.155320 8054 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:29:17.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.159957 8054 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:29:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.161612 8054 tablet_server_main.cc:90] Starting tablet server... 01:29:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.306951 8054 rpc_server.cc:205] RPC server started. Bound to: 127.7.98.195:50682 01:29:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.308550 8054 webserver.cc:175] Starting webserver on 127.7.98.195:0 01:29:17.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.308903 8054 webserver.cc:186] Document root disabled 01:29:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.310920 8054 webserver.cc:313] Webserver started. Bound to: http://127.7.98.195:47167/ 01:29:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.312006 8054 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster6140261054714180238/ts-2/data/info.pb 01:29:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.314851 7563 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 8054 01:29:17.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.360637 8054 tablet_server_main.cc:93] Tablet server successfully started. 01:29:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.422253 8183 heartbeater.cc:345] Connected to a master server at 127.7.98.254:35261 01:29:17.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.422950 8183 heartbeater.cc:426] Registering TS with master... 01:29:17.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.424935 8180 heartbeater.cc:345] Connected to a master server at 127.7.98.252:36904 01:29:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.425330 8180 heartbeater.cc:426] Registering TS with master... 01:29:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.426240 8180 heartbeater.cc:475] Master 127.7.98.252:36904 requested a full tablet report, sending... 01:29:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.431094 7733 ts_manager.cc:98] Registered new tserver with Master: c0d03701d5354bceb19d248d017859bf (127.7.98.195:50682) 01:29:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.425047 8183 heartbeater.cc:475] Master 127.7.98.254:35261 requested a full tablet report, sending... 01:29:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.445674 7603 ts_manager.cc:98] Registered new tserver with Master: c0d03701d5354bceb19d248d017859bf (127.7.98.195:50682) 01:29:17.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.428129 8181 heartbeater.cc:345] Connected to a master server at 127.7.98.253:57750 01:29:17.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.450382 8181 heartbeater.cc:426] Registering TS with master... 01:29:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.451117 8181 heartbeater.cc:475] Master 127.7.98.253:57750 requested a full tablet report, sending... 01:29:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.453686 7667 ts_manager.cc:98] Registered new tserver with Master: c0d03701d5354bceb19d248d017859bf (127.7.98.195:50682) 01:29:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.455893 7563 external_mini_cluster.cc:623] 3 TS(s) registered with all masters 01:29:17.456 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: 01:29:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.456251 7667 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.7.98.195:40088 01:29:17.456 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: get_masters { } 01:29:17.458 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.7.98.254" port: 35261 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.7.98.253" port: 57750 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.7.98.252" port: 36904 } } } 01:29:17.459 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: get_tservers { } 01:29:17.462 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.7.98.193" port: 54247 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.7.98.194" port: 50881 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.7.98.195" port: 50682 } } } 01:29:17.462 [INFO - Test worker] (KuduTestHarness.java:142) Creating a new Kudu client... 01:29:17.465 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:29:17.470 [DEBUG - New I/O worker #2161] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.98.254 01:29:17.472 [DEBUG - New I/O worker #2162] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.98.253 01:29:17.474 [DEBUG - New I/O worker #2163] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.7.98.252 01:29:17.484 [DEBUG - New I/O worker #2162] (Negotiator.java:764) Authenticated connection [id: 0x7019b471, /127.7.98.253:49560 => /127.7.98.253:57750] using SASL/PLAIN 01:29:17.488 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:29:17.488 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=22), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.98.254:35261, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.98.253:57750, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.98.252:36904, [22ms] Sub RPC ConnectToMaster: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:17.488 [DEBUG - New I/O worker #2163] (Negotiator.java:764) Authenticated connection [id: 0xe0414a38, /127.7.98.252:59269 => /127.7.98.252:36904] using SASL/PLAIN 01:29:17.489 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@1127442047 returned Deferred@1192032961(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after retry RPC@1127442047), errback=(continuation of Deferred@34499652 after retry RPC@1127442047)), so the following Deferred is getting paused: Deferred@34499652(state=PAUSED, result=Deferred@1192032961, callback=org.apache.kudu.client.AsyncKuduClient$1@6b9e1cad -> org.apache.kudu.client.AsyncKuduClient$2@33c68f3f -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 01:29:17.490 [DEBUG - New I/O worker #2161] (Negotiator.java:764) Authenticated connection [id: 0x3b181cc1, /127.7.98.254:37581 => /127.7.98.254:35261] using SASL/PLAIN 01:29:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.490764 7667 catalog_manager.cc:1386] Servicing CreateTable request from {username='jenkins'} at 127.7.98.253:49560: 01:29:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "testMultipleSessions" 01:29:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) schema { 01:29:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: true 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column1_i" 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column2_i" 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:29:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column3_s" 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: STRING 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: true 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) encoding: DICT_ENCODING 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) compression: LZ4 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 4096 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column4_b" 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: BOOL 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) partition_schema { 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) range_schema { 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:29:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:17.507971 7667 catalog_manager.cc:1537] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table testMultipleSessions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 01:29:17.536 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@6b9e1cad@1805524141 returned Deferred@331129204(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@6221930b -> (continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$1@6b9e1cad@1805524141), errback=passthrough -> (continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$1@6b9e1cad@1805524141)), so the following Deferred is getting paused: Deferred@34499652(state=PAUSED, result=Deferred@331129204, callback=org.apache.kudu.client.AsyncKuduClient$2@33c68f3f -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:29:17.538 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:761) Opened table 513a82d7452340f59267d8e414e9cce6 01:29:17.538 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815)), so the following Deferred is getting paused: Deferred@34499652(state=PAUSED, result=Deferred@2042728598, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 01:29:17.553 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478)), so the following Deferred is getting paused: Deferred@146337372(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.570571 7851 tablet_service.cc:870] Processing CreateTablet for tablet a6cc3ab4fc85448a835374e9531f2c29 (table=testMultipleSessions [id=513a82d7452340f59267d8e414e9cce6]), partition=RANGE (key) PARTITION UNBOUNDED 01:29:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.571653 7851 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet a6cc3ab4fc85448a835374e9531f2c29. 1 dirs total, 0 dirs full, 0 dirs failed 01:29:17.578 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763)), so the following Deferred is getting paused: Deferred@333444336(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.581733 7986 tablet_service.cc:870] Processing CreateTablet for tablet a6cc3ab4fc85448a835374e9531f2c29 (table=testMultipleSessions [id=513a82d7452340f59267d8e414e9cce6]), partition=RANGE (key) PARTITION UNBOUNDED 01:29:17.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.582984 7986 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet a6cc3ab4fc85448a835374e9531f2c29. 1 dirs total, 0 dirs full, 0 dirs failed 01:29:17.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.585925 8117 tablet_service.cc:870] Processing CreateTablet for tablet a6cc3ab4fc85448a835374e9531f2c29 (table=testMultipleSessions [id=513a82d7452340f59267d8e414e9cce6]), partition=RANGE (key) PARTITION UNBOUNDED 01:29:17.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.586952 8117 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet a6cc3ab4fc85448a835374e9531f2c29. 1 dirs total, 0 dirs full, 0 dirs failed 01:29:17.590 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122)), so the following Deferred is getting paused: Deferred@1747453885(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.595316 7851 ts_tablet_manager.cc:1175] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Registered tablet (data state: TABLET_DATA_READY) 01:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.597432 8221 ts_tablet_manager.cc:1030] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Bootstrapping tablet 01:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.598649 8221 tablet_bootstrap.cc:438] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Bootstrap starting. 01:29:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.603312 8221 tablet_bootstrap.cc:592] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: No blocks or log segments found. Creating new log. 01:29:17.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.604861 7986 ts_tablet_manager.cc:1175] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Registered tablet (data state: TABLET_DATA_READY) 01:29:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.605909 8221 log.cc:525] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Log is configured to *not* fsync() on all Append() calls 01:29:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.607455 8222 ts_tablet_manager.cc:1030] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Bootstrapping tablet 01:29:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.608342 8222 tablet_bootstrap.cc:438] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Bootstrap starting. 01:29:17.609 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046)), so the following Deferred is getting paused: Deferred@505794100(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.609505 8117 ts_tablet_manager.cc:1175] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Registered tablet (data state: TABLET_DATA_READY) 01:29:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.612911 8224 ts_tablet_manager.cc:1030] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Bootstrapping tablet 01:29:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.613101 8222 tablet_bootstrap.cc:592] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: No blocks or log segments found. Creating new log. 01:29:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.614176 8224 tablet_bootstrap.cc:438] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Bootstrap starting. 01:29:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.615664 8221 tablet_bootstrap.cc:438] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: No bootstrap required, opened a new log 01:29:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.616113 8221 ts_tablet_manager.cc:1047] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Time spent bootstrapping tablet: real 0.018s user 0.011s sys 0.003s 01:29:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.615980 8222 log.cc:525] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Log is configured to *not* fsync() on all Append() calls 01:29:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.619668 8224 tablet_bootstrap.cc:592] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: No blocks or log segments found. Creating new log. 01:29:17.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.620532 8222 tablet_bootstrap.cc:438] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: No bootstrap required, opened a new log 01:29:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.621127 8222 ts_tablet_manager.cc:1047] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Time spent bootstrapping tablet: real 0.013s user 0.008s sys 0.003s 01:29:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.622573 8224 log.cc:525] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Log is configured to *not* fsync() on all Append() calls 01:29:17.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.627326 8224 tablet_bootstrap.cc:438] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: No bootstrap required, opened a new log 01:29:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.627995 8224 ts_tablet_manager.cc:1047] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Time spent bootstrapping tablet: real 0.014s user 0.006s sys 0.006s 01:29:17.630 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733)), so the following Deferred is getting paused: Deferred@2143600464(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.632525 8221 raft_consensus.cc:340] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.633299 8221 raft_consensus.cc:366] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:29:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.633579 8221 raft_consensus.cc:705] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1170895397248e39a195e2b9e398f75, State: Initialized, Role: FOLLOWER 01:29:17.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.634376 8221 consensus_queue.cc:229] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.638499 8221 ts_tablet_manager.cc:1075] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Time spent starting tablet: real 0.022s user 0.018s sys 0.002s 01:29:17.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.648440 8222 raft_consensus.cc:340] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.649569 8222 raft_consensus.cc:366] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:29:17.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.649845 8222 raft_consensus.cc:705] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a82b579bdd3d49869b0e45fd42da09f5, State: Initialized, Role: FOLLOWER 01:29:17.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.650815 8222 consensus_queue.cc:229] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.653916 8050 heartbeater.cc:467] Master 127.7.98.253:57750 was elected leader, sending a full tablet report... 01:29:17.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.656050 8222 ts_tablet_manager.cc:1075] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Time spent starting tablet: real 0.035s user 0.026s sys 0.001s 01:29:17.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.656980 8224 raft_consensus.cc:340] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.658121 8224 raft_consensus.cc:366] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:29:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.658499 8224 raft_consensus.cc:705] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c0d03701d5354bceb19d248d017859bf, State: Initialized, Role: FOLLOWER 01:29:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.659528 8224 consensus_queue.cc:229] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.662436 8181 heartbeater.cc:467] Master 127.7.98.253:57750 was elected leader, sending a full tablet report... 01:29:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.664548 8224 ts_tablet_manager.cc:1075] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Time spent starting tablet: real 0.036s user 0.024s sys 0.003s 01:29:17.670 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733) -> (continuation of Deferred@864694405 after org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733) -> (continuation of Deferred@864694405 after org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938)), so the following Deferred is getting paused: Deferred@864694405(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.689 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@2e9117ae@781260718 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733) -> (continuation of Deferred@864694405 after org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938) -> (continuation of Deferred@1962214163 after org.apache.kudu.client.AsyncKuduClient$15@2e9117ae@781260718), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733) -> (continuation of Deferred@864694405 after org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938) -> (continuation of Deferred@1962214163 after org.apache.kudu.client.AsyncKuduClient$15@2e9117ae@781260718)), so the following Deferred is getting paused: Deferred@1962214163(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.713928 8229 raft_consensus.cc:472] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:29:17.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.714772 8229 raft_consensus.cc:494] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.718746 8229 leader_election.cc:296] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a82b579bdd3d49869b0e45fd42da09f5 (127.7.98.194:50881), a1170895397248e39a195e2b9e398f75 (127.7.98.193:54247) 01:29:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.731171 7871 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "a6cc3ab4fc85448a835374e9531f2c29" candidate_uuid: "c0d03701d5354bceb19d248d017859bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1170895397248e39a195e2b9e398f75" is_pre_election: true 01:29:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.731459 8006 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "a6cc3ab4fc85448a835374e9531f2c29" candidate_uuid: "c0d03701d5354bceb19d248d017859bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a82b579bdd3d49869b0e45fd42da09f5" is_pre_election: true 01:29:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.732036 8006 raft_consensus.cc:2346] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c0d03701d5354bceb19d248d017859bf in term 0. 01:29:17.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.732043 7871 raft_consensus.cc:2346] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c0d03701d5354bceb19d248d017859bf in term 0. 01:29:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.733901 8069 leader_election.cc:310] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a1170895397248e39a195e2b9e398f75, c0d03701d5354bceb19d248d017859bf; no voters: 01:29:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.735008 8229 raft_consensus.cc:2640] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Leader pre-election won for term 1 01:29:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.735370 8229 raft_consensus.cc:472] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:29:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.735656 8229 raft_consensus.cc:2886] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 0 FOLLOWER]: Advancing to term 1 01:29:17.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.741398 8229 raft_consensus.cc:494] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.743324 8229 leader_election.cc:296] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [CANDIDATE]: Term 1 election: Requested vote from peers a82b579bdd3d49869b0e45fd42da09f5 (127.7.98.194:50881), a1170895397248e39a195e2b9e398f75 (127.7.98.193:54247) 01:29:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.745707 7871 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "a6cc3ab4fc85448a835374e9531f2c29" candidate_uuid: "c0d03701d5354bceb19d248d017859bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1170895397248e39a195e2b9e398f75" 01:29:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.745904 8006 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "a6cc3ab4fc85448a835374e9531f2c29" candidate_uuid: "c0d03701d5354bceb19d248d017859bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a82b579bdd3d49869b0e45fd42da09f5" 01:29:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.746264 7871 raft_consensus.cc:2886] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 0 FOLLOWER]: Advancing to term 1 01:29:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.746417 8006 raft_consensus.cc:2886] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 0 FOLLOWER]: Advancing to term 1 01:29:17.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.752518 8006 raft_consensus.cc:2346] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c0d03701d5354bceb19d248d017859bf in term 1. 01:29:17.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.753123 7871 raft_consensus.cc:2346] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c0d03701d5354bceb19d248d017859bf in term 1. 01:29:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.754249 8067 leader_election.cc:310] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a82b579bdd3d49869b0e45fd42da09f5, c0d03701d5354bceb19d248d017859bf; no voters: 01:29:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.755779 8229 raft_consensus.cc:2640] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 1 FOLLOWER]: Leader election won for term 1 01:29:17.757 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@73dbc35@121486389 returned Deferred@2042728598(state=PENDING, result=null, callback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733) -> (continuation of Deferred@864694405 after org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938) -> (continuation of Deferred@1962214163 after org.apache.kudu.client.AsyncKuduClient$15@2e9117ae@781260718) -> (continuation of Deferred@1699854927 after org.apache.kudu.client.AsyncKuduClient$15@73dbc35@121486389), errback=(continuation of Deferred@34499652 after org.apache.kudu.client.AsyncKuduClient$2@33c68f3f@868650815) -> (continuation of Deferred@146337372 after org.apache.kudu.client.AsyncKuduClient$15@7e7947e6@2121877478) -> (continuation of Deferred@333444336 after org.apache.kudu.client.AsyncKuduClient$15@666f8d8b@1718586763) -> (continuation of Deferred@1747453885 after org.apache.kudu.client.AsyncKuduClient$15@27adc71a@665700122) -> (continuation of Deferred@505794100 after org.apache.kudu.client.AsyncKuduClient$15@72b1dce6@1924259046) -> (continuation of Deferred@2143600464 after org.apache.kudu.client.AsyncKuduClient$15@57f94ffd@1475956733) -> (continuation of Deferred@864694405 after org.apache.kudu.client.AsyncKuduClient$15@3ee0aa2@65931938) -> (continuation of Deferred@1962214163 after org.apache.kudu.client.AsyncKuduClient$15@2e9117ae@781260718) -> (continuation of Deferred@1699854927 after org.apache.kudu.client.AsyncKuduClient$15@73dbc35@121486389)), so the following Deferred is getting paused: Deferred@1699854927(state=PAUSED, result=Deferred@2042728598, callback=, errback=) 01:29:17.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.762522 8229 raft_consensus.cc:667] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [term 1 LEADER]: Becoming Leader. State: Replica: c0d03701d5354bceb19d248d017859bf, State: Running, Role: LEADER 01:29:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.763595 8229 consensus_queue.cc:206] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } } 01:29:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:17.774214 7667 catalog_manager.cc:3791] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf reported cstate change: term changed from 0 to 1, leader changed from to c0d03701d5354bceb19d248d017859bf (127.7.98.195). New cstate: current_term: 1 leader_uuid: "c0d03701d5354bceb19d248d017859bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c0d03701d5354bceb19d248d017859bf" member_type: VOTER last_known_addr { host: "127.7.98.195" port: 50682 } health_report { overall_health: HEALTHY } } } 01:29:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:17.709781 8055 tablet.cc:1783] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:29:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:17.855641 8187 tablet.cc:1783] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:29:17.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:17.863118 7922 tablet.cc:1783] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:29:18.014 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.014 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.014 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [3ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.016 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@535462863 returned Deferred@1968986131(state=PENDING, result=null, callback=(continuation of Deferred@498534197 after retry RPC@535462863), errback=(continuation of Deferred@498534197 after retry RPC@535462863)), so the following Deferred is getting paused: Deferred@498534197(state=PAUSED, result=Deferred@1968986131, callback=org.apache.kudu.client.AsyncKuduSession$2@199ab22 -> passthrough -> wakeup thread pool-486-thread-1, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4aa6b425 -> wakeup thread pool-486-thread-1) 01:29:18.016 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.016 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.016 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [5ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.017 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@1563287970 returned Deferred@2022099869(state=PENDING, result=null, callback=(continuation of Deferred@873119508 after retry RPC@1563287970), errback=(continuation of Deferred@873119508 after retry RPC@1563287970)), so the following Deferred is getting paused: Deferred@873119508(state=PAUSED, result=Deferred@2022099869, callback=org.apache.kudu.client.AsyncKuduSession$2@4b72222c -> passthrough -> wakeup thread pool-486-thread-3, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7960fa11 -> wakeup thread pool-486-thread-3) 01:29:18.020 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.020 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.021 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=8), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [8ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.021 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@938740867 returned Deferred@1449650038(state=PENDING, result=null, callback=(continuation of Deferred@215928020 after retry RPC@938740867), errback=(continuation of Deferred@215928020 after retry RPC@938740867)), so the following Deferred is getting paused: Deferred@215928020(state=PAUSED, result=Deferred@1449650038, callback=org.apache.kudu.client.AsyncKuduSession$2@72101f2c -> passthrough -> wakeup thread pool-486-thread-5, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@651785f9 -> wakeup thread pool-486-thread-5) 01:29:18.021 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.021 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.022 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=7), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [7ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.022 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@203588218 returned Deferred@1994413894(state=PENDING, result=null, callback=(continuation of Deferred@75222985 after retry RPC@203588218), errback=(continuation of Deferred@75222985 after retry RPC@203588218)), so the following Deferred is getting paused: Deferred@75222985(state=PAUSED, result=Deferred@1994413894, callback=org.apache.kudu.client.AsyncKuduSession$2@5d9abaf2 -> passthrough -> wakeup thread pool-486-thread-4, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45b3c269 -> wakeup thread pool-486-thread-4) 01:29:18.022 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.022 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.023 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=9), Traces: [0ms] refreshing cache from master, [2ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [9ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.027 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@832432126 returned Deferred@1799517059(state=PENDING, result=null, callback=(continuation of Deferred@1661409587 after retry RPC@832432126), errback=(continuation of Deferred@1661409587 after retry RPC@832432126)), so the following Deferred is getting paused: Deferred@1661409587(state=PAUSED, result=Deferred@1799517059, callback=org.apache.kudu.client.AsyncKuduSession$2@38fc3fb0 -> passthrough -> wakeup thread pool-486-thread-6, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@66008d46 -> wakeup thread pool-486-thread-6) 01:29:18.027 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.027 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.027 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=14), Traces: [0ms] refreshing cache from master, [2ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [14ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.028 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@514208878 returned Deferred@1161000968(state=PENDING, result=null, callback=(continuation of Deferred@485215905 after retry RPC@514208878), errback=(continuation of Deferred@485215905 after retry RPC@514208878)), so the following Deferred is getting paused: Deferred@485215905(state=PAUSED, result=Deferred@1161000968, callback=org.apache.kudu.client.AsyncKuduSession$2@7ee30ac7 -> passthrough -> wakeup thread pool-486-thread-7, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@51c95431 -> wakeup thread pool-486-thread-7) 01:29:18.028 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:2118) Learned about tablet a6cc3ab4fc85448a835374e9531f2c29 for table 'testMultipleSessions' with partition [, ) 01:29:18.028 [DEBUG - New I/O worker #2162] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a6cc3ab4fc85448a835374e9531f2c29@[a1170895397248e39a195e2b9e398f75(127.7.98.193:54247),a82b579bdd3d49869b0e45fd42da09f5(127.7.98.194:50881),c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)[L]]}] 01:29:18.029 [DEBUG - New I/O worker #2162] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=16), Traces: [0ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.7.98.253:57750, [16ms] Sub RPC GetTableLocations: received response from server master-127.7.98.253:57750: OK, deferred=null) after lookup 01:29:18.029 [DEBUG - New I/O worker #2162] (Deferred.java:1330) callback=retry RPC@266634552 returned Deferred@1303960739(state=PENDING, result=null, callback=(continuation of Deferred@736838369 after retry RPC@266634552), errback=(continuation of Deferred@736838369 after retry RPC@266634552)), so the following Deferred is getting paused: Deferred@736838369(state=PAUSED, result=Deferred@1303960739, callback=org.apache.kudu.client.AsyncKuduSession$2@3018f371 -> passthrough -> wakeup thread pool-486-thread-2, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1bfe6d0f -> wakeup thread pool-486-thread-2) 01:29:18.031 [DEBUG - New I/O worker #2164] (Negotiator.java:764) Authenticated connection [id: 0x2134b643, /127.7.98.195:37877 => /127.7.98.195:50682] using TOKEN/null 01:29:18.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:18.137094 8006 raft_consensus.cc:1181] T a6cc3ab4fc85448a835374e9531f2c29 P a82b579bdd3d49869b0e45fd42da09f5 [term 1 FOLLOWER]: Refusing update from remote peer c0d03701d5354bceb19d248d017859bf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:29:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:18.138455 7871 raft_consensus.cc:1181] T a6cc3ab4fc85448a835374e9531f2c29 P a1170895397248e39a195e2b9e398f75 [term 1 FOLLOWER]: Refusing update from remote peer c0d03701d5354bceb19d248d017859bf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:29:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:18.139232 8229 consensus_queue.cc:984] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "a82b579bdd3d49869b0e45fd42da09f5" member_type: VOTER last_known_addr { host: "127.7.98.194" port: 50881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:29:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:18.141203 8230 consensus_queue.cc:984] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1170895397248e39a195e2b9e398f75" member_type: VOTER last_known_addr { host: "127.7.98.193" port: 54247 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s 01:29:18.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:29:18.893398 8293 mvcc.cc:203] Tried to move safe_time back from 6351128817184423936 to 6351128812611153920. Current Snapshot: MvccSnapshot[committed={T|T < 6351128817184423936 or (T in {6351128817184423936})}] 01:29:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:24.506582 7588 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 7576 01:29:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:24.507994 7588 debug-util.cc:397] Leaking SignalData structure 0x7b080000c660 after lost signal to thread 7634 01:29:27.508 [DEBUG - New I/O worker #2163] (Connection.java:429) [peer master-127.7.98.252:36904(127.7.98.252:36904)] encountered a read timeout; closing the channel 01:29:27.508 [DEBUG - New I/O worker #2163] (Connection.java:688) [peer master-127.7.98.252:36904(127.7.98.252:36904)] cleaning up while in state READY due to: [peer master-127.7.98.252:36904(127.7.98.252:36904)] encountered a read timeout; closing the channel 01:29:27.508 [DEBUG - New I/O worker #2161] (Connection.java:429) [peer master-127.7.98.254:35261(127.7.98.254:35261)] encountered a read timeout; closing the channel 01:29:27.509 [DEBUG - New I/O worker #2161] (Connection.java:688) [peer master-127.7.98.254:35261(127.7.98.254:35261)] cleaning up while in state READY due to: [peer master-127.7.98.254:35261(127.7.98.254:35261)] encountered a read timeout; closing the channel 01:29:28.048 [DEBUG - New I/O worker #2162] (Connection.java:429) [peer master-127.7.98.253:57750(127.7.98.253:57750)] encountered a read timeout; closing the channel 01:29:28.048 [DEBUG - New I/O worker #2162] (Connection.java:688) [peer master-127.7.98.253:57750(127.7.98.253:57750)] cleaning up while in state READY due to: [peer master-127.7.98.253:57750(127.7.98.253:57750)] encountered a read timeout; closing the channel 01:29:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:33.594795 7652 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 7640 01:29:33.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:33.596129 7652 debug-util.cc:397] Leaking SignalData structure 0x7b0800079400 after lost signal to thread 7698 01:29:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:48.042297 7806 debug-util.cc:397] Leaking SignalData structure 0x7b0800029080 after lost signal to thread 7782 01:29:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:29:48.044309 7806 debug-util.cc:397] Leaking SignalData structure 0x7b080009e380 after lost signal to thread 7912 01:30:17.468 [DEBUG - Time-limited test] (Deferred.java:1189) While joining Deferred@1824806092(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$2@49cb8c3e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1642cb5 -> wakeup thread Time-limited test): interrupted 01:30:17.470 [DEBUG - Test worker] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:30:17.470 [DEBUG - New I/O worker #2164] (Connection.java:688) [peer c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682)] cleaning up while in state READY due to: connection disconnected 01:30:17.470 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.471 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.471 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.472 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.472 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.472 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.472 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.473 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.474 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.475 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.482 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:17.475751 7563 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 7767 01:30:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.477200 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5543, attempt_no=1}) could send its response 01:30:17.484 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.484 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.485 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.486 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.486 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.487 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.488 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.488 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.488 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.492250 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5544, attempt_no=1}) could send its response 01:30:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.493500 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5545, attempt_no=1}) could send its response 01:30:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.505339 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5546, attempt_no=1}) could send its response 01:30:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.505908 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5547, attempt_no=1}) could send its response 01:30:17.507 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.507 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.508 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.508 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.508 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.508 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.509 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.509 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.509 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.509 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.510 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.510 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.510 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.514 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.514 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.516021 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5548, attempt_no=1}) could send its response 01:30:17.522 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.522 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.522 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.525 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.524926 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5549, attempt_no=1}) could send its response 01:30:17.525 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.525 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.528 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.529 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.529 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.533 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.533 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.533 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.536461 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5550, attempt_no=1}) could send its response 01:30:17.540 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.540 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.540 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.545 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.545 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.545 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.549 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.549 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.549 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.552266 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5551, attempt_no=1}) could send its response 01:30:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.552731 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5552, attempt_no=1}) could send its response 01:30:17.558 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.558 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.559 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.560 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.561 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.561 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.564 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.564637 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5553, attempt_no=1}) could send its response 01:30:17.565 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.565 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.569 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.569 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.570 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.576 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.577 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.577 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.581 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.584 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.584 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.585268 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5554, attempt_no=1}) could send its response 01:30:17.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.585794 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5555, attempt_no=1}) could send its response 01:30:17.592 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.592 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.593 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.605 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.608892 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5556, attempt_no=1}) could send its response 01:30:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.609428 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5557, attempt_no=1}) could send its response 01:30:17.615 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.616 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.616289 7714 connection.cc:647] server connection from 127.7.98.193:35165 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.193:35165): Connection reset by peer (error 104) 01:30:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.619772 7648 connection.cc:647] server connection from 127.7.98.193:54683 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.193:54683): Connection reset by peer (error 104) 01:30:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.622020 7584 connection.cc:647] server connection from 127.7.98.193:32803 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.193:32803): Connection reset by peer (error 104) 01:30:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:17.624644 7563 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 7918 01:30:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.624811 8069 connection.cc:647] client connection to 127.7.98.193:54247 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.193:54247): Connection reset by peer (error 104) 01:30:17.625 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.625222 8069 consensus_peers.cc:457] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf -> Peer a1170895397248e39a195e2b9e398f75 (127.7.98.193:54247): Couldn't send request to peer a1170895397248e39a195e2b9e398f75. Status: Network error: failed to read from TLS socket (remote: 127.7.98.193:54247): Connection reset by peer (error 104). This is attempt 1: this message will repeat every 5th retry. 01:30:17.626 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.626 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.630458 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5558, attempt_no=1}) could send its response 01:30:17.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.630991 8071 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.7.98.195:37877 (ReqId={client: 7be44d99352d4991bcc106c69f63d6f5, seq_no=5560, attempt_no=1}) could send its response 01:30:17.634 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.634 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.634 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.641 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.641 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.641 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.642 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.642 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.642 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.642 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.642 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.643 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.643 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.643 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.643 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.643 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.644 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.644 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.644 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.644 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.644 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.645 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.645 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.645 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.645 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.645 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.646 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.646 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.646 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.646 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.647 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.647 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.647 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.647 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.647 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.647 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.648 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.648 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.648 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.648 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.648 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.649 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.649 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.649 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.649 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.650 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.650 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.650 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.650 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.651 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.651 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.652 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.652 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.652 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.653 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.653 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.653 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.654 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.654 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.654 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.657 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.657 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.657 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.660 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.660 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.660 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.663 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.663 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.664 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.667 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.667 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.667 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.669 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.669 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.669 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.671 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.671 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.672 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.674 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.674 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.675 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.678 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.678 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.678 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.681279 7647 connection.cc:647] server connection from 127.7.98.194:39109 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.194:39109): Connection reset by peer (error 104) 01:30:17.682 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.682 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.682785 7712 connection.cc:647] server connection from 127.7.98.194:57226 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.194:57226): Connection reset by peer (error 104) 01:30:17.683 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.683661 7587 connection.cc:647] server connection from 127.7.98.194:42962 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.194:42962): Connection reset by peer (error 104) 01:30:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.684684 8067 connection.cc:647] client connection to 127.7.98.194:50881 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:30:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.685065 8067 consensus_peers.cc:457] T a6cc3ab4fc85448a835374e9531f2c29 P c0d03701d5354bceb19d248d017859bf -> Peer a82b579bdd3d49869b0e45fd42da09f5 (127.7.98.194:50881): Couldn't send request to peer a82b579bdd3d49869b0e45fd42da09f5. Status: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104). This is attempt 1: this message will repeat every 5th retry. 01:30:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.685866 8067 connection.cc:320] Error closing socket: Network error: TlsSocket::Close: Broken pipe 01:30:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:17.686486 7563 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 8054 01:30:17.687 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.688 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.688 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.691 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.691 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.692 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.695 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.695 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.696 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.699 [INFO - New I/O worker #2164] (AsyncKuduClient.java:1970) Invalidating location c0d03701d5354bceb19d248d017859bf(127.7.98.195:50682) for tablet a6cc3ab4fc85448a835374e9531f2c29: connection disconnected 01:30:17.699 [DEBUG - New I/O worker #2164] (RemoteTablet.java:131) tablet a6cc3ab4fc85448a835374e9531f2c29 already removed ts c0d03701d5354bceb19d248d017859bf, size left is 2 01:30:17.699 [WARN - New I/O worker #2164] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:30:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.728972 7713 connection.cc:647] server connection from 127.7.98.195:35573 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.195:35573): Connection reset by peer (error 104) 01:30:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:17.729552 7563 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 7575 01:30:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:17.729176 7648 connection.cc:647] server connection from 127.7.98.195:40088 recv error: Network error: failed to read from TLS socket (remote: 127.7.98.195:40088): Connection reset by peer (error 104) 01:30:17.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:17.750463 7563 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 7637 01:30:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:17.769189 7563 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 7701 01:30:18.822 [ERROR - Test worker] (RetryRule.java:80) testMultipleSessions(org.apache.kudu.client.ITClientStress): failed attempt 3 org.junit.runners.model.TestTimedOutException: test timed out after 60000 milliseconds at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.stumbleupon.async.Deferred.doJoin(Deferred.java:1136) at com.stumbleupon.async.Deferred.join(Deferred.java:1019) at org.apache.kudu.client.KuduSession.apply(KuduSession.java:81) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:163) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:150) at org.apache.kudu.client.ITClientStress$1.run(ITClientStress.java:80) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:2022) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at org.apache.kudu.client.ITClientStress.runTasks(ITClientStress.java:86) at org.apache.kudu.client.ITClientStress.testMultipleSessions(ITClientStress.java:147) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 01:30:18.824 [INFO - Test worker] (KuduTestHarness.java:140) Creating a new MiniKuduCluster... 01:30:18.824 [INFO - Test worker] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /data/somelongdirectorytoavoidrpathissues/src/kudutest 01:30:18.824 [INFO - Test worker] (KuduBinaryLocator.java:78) Using Kudu binary directory specified by system property 'kuduBinDir': /data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin 01:30:18.825 [INFO - Test worker] (MiniKuduCluster.java:187) Starting process: [/data/somelongdirectorytoavoidrpathissues/src/kudu/java/../build/latest/bin/kudu, test, mini_cluster, --serialization=pb] 01:30:18.827 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: create_cluster { num_masters: 3 num_tservers: 3 enable_kerberos: false cluster_root: "/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278" hms_mode: NONE mini_kdc_options { } } 01:30:19.076 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: 01:30:19.076 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: start_cluster { } 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.105873 8582 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/wal 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.190:34778 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.190 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/logs 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data/info.pb 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:30:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 with env {} 01:30:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:30:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.312409 8590 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:30:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.313094 8590 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:30:19.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.324841 8590 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:30:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.325019 8590 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:30:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.325414 8590 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:30:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.325651 8590 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:30:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.352305 8590 master_main.cc:73] Master server non-default flags: 01:30:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data 01:30:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/wal 01:30:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:30:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.190:34778 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data/info.pb 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.190 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/logs 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:30:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:30:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:30:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.369228 8590 master_main.cc:80] Initializing master server... 01:30:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.372021 8590 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 189378us 01:30:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.372967 8590 fs_manager.cc:263] Metadata directory not provided 01:30:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.373143 8590 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:30:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.373466 8590 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:30:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.373606 8590 server_base.cc:434] Attempting to create new FS layout instead 01:30:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.378599 8590 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data/instance: 01:30:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "4ef7405012c144928614dbf254037540" 01:30:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:19 on quasar-aecznm-1.vpc.cloudera.com" 01:30:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.379580 8590 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/wal/instance: 01:30:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "4ef7405012c144928614dbf254037540" 01:30:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:19 on quasar-aecznm-1.vpc.cloudera.com" 01:30:19.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.398484 8590 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.009s sys 0.007s 01:30:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.400174 8590 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:30:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.401978 8590 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:30:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.412003 8590 fs_manager.cc:419] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s 01:30:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.412170 8590 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/wal 01:30:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "4ef7405012c144928614dbf254037540" 01:30:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:19 on quasar-aecznm-1.vpc.cloudera.com" 01:30:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.412505 8590 fs_report.cc:352] FS layout report 01:30:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/wal 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/wal 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data/data 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:30:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:30:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.593395 8590 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:30:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.598239 8590 master_main.cc:83] Starting Master server... 01:30:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.721637 8590 rpc_server.cc:205] RPC server started. Bound to: 127.8.97.190:34778 01:30:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.722743 8590 webserver.cc:175] Starting webserver on 127.8.97.190:0 01:30:19.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.722918 8590 webserver.cc:186] Document root disabled 01:30:19.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.725500 8590 webserver.cc:313] Webserver started. Bound to: http://127.8.97.190:38255/ 01:30:19.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.726541 8590 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-0/data/info.pb 01:30:19.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.731489 8651 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:30:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.735399 8582 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 8590 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.736444 8582 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/wal 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:30:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.189:35516 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.189 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/logs 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data/info.pb 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:30:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 with env {} 01:30:19.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.748011 8651 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } has no permanent_uuid. Determining permanent_uuid... 01:30:19.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.776521 8651 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } has no permanent_uuid. Determining permanent_uuid... 01:30:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.778823 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.189:35516: Network error: Client connection negotiation failed: client connection to 127.8.97.189:35516: connect: Connection refused (error 111) 01:30:19.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.798496 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } attempt: 1 01:30:19.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.800158 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.189:35516: Network error: Client connection negotiation failed: client connection to 127.8.97.189:35516: connect: Connection refused (error 111) 01:30:19.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.848544 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } attempt: 2 01:30:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.850329 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.189:35516: Network error: Client connection negotiation failed: client connection to 127.8.97.189:35516: connect: Connection refused (error 111) 01:30:19.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:19.961762 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } attempt: 3 01:30:19.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:30:19.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.962602 8652 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:30:19.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.963197 8652 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:30:19.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.963614 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.189:35516: Network error: Client connection negotiation failed: client connection to 127.8.97.189:35516: connect: Connection refused (error 111) 01:30:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.975147 8652 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:30:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.975304 8652 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:30:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.975455 8652 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:30:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:19.975620 8652 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.001736 8652 master_main.cc:73] Master server non-default flags: 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/wal 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.189:35516 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data/info.pb 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.189 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/logs 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:30:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:30:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:30:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:30:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:30:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:30:20.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.018597 8652 master_main.cc:80] Initializing master server... 01:30:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.021860 8652 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 189878us 01:30:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.022891 8652 fs_manager.cc:263] Metadata directory not provided 01:30:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.023049 8652 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:30:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.023430 8652 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:30:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.023571 8652 server_base.cc:434] Attempting to create new FS layout instead 01:30:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.029902 8652 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data/instance: 01:30:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "28d5e62cff544540986a404b148f2a7f" 01:30:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:20 on quasar-aecznm-1.vpc.cloudera.com" 01:30:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.030874 8652 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/wal/instance: 01:30:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "28d5e62cff544540986a404b148f2a7f" 01:30:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:20 on quasar-aecznm-1.vpc.cloudera.com" 01:30:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.048910 8652 fs_manager.cc:503] Time spent creating directory manager: real 0.017s user 0.010s sys 0.006s 01:30:20.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.050506 8652 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:30:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.052227 8652 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:30:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.062098 8652 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s 01:30:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.062247 8652 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/wal 01:30:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "28d5e62cff544540986a404b148f2a7f" 01:30:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:20 on quasar-aecznm-1.vpc.cloudera.com" 01:30:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.062547 8652 fs_report.cc:352] FS layout report 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/wal 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/wal 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data/data 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:30:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:30:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.140007 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } attempt: 4 01:30:20.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.142375 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.189:35516: Network error: Client connection negotiation failed: client connection to 127.8.97.189:35516: connect: Connection refused (error 111) 01:30:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.256212 8652 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:30:20.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.259904 8652 master_main.cc:83] Starting Master server... 01:30:20.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.377771 8652 rpc_server.cc:205] RPC server started. Bound to: 127.8.97.189:35516 01:30:20.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.378909 8652 webserver.cc:175] Starting webserver on 127.8.97.189:0 01:30:20.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.379037 8652 webserver.cc:186] Document root disabled 01:30:20.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.381572 8652 webserver.cc:313] Webserver started. Bound to: http://127.8.97.189:51137/ 01:30:20.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.383085 8652 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-1/data/info.pb 01:30:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.387967 8715 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:30:20.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.391350 8582 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 8652 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.392343 8582 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/wal 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=localhost 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:30:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.188:45488 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.188 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/logs 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data/info.pb 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:30:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 with env {} 01:30:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.409345 8715 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } has no permanent_uuid. Determining permanent_uuid... 01:30:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.431650 8715 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } has no permanent_uuid. Determining permanent_uuid... 01:30:20.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.445842 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } attempt: 5 01:30:20.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.447595 8715 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } has no permanent_uuid. Determining permanent_uuid... 01:30:20.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.449411 8715 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.454607 8651 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } has no permanent_uuid. Determining permanent_uuid... 01:30:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.455979 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.468889 8715 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 1 01:30:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.470285 8715 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.494375 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 1 01:30:20.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.496009 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.518663 8715 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 2 01:30:20.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.520349 8715 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.544385 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 2 01:30:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.546602 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:30:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.621515 8716 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:30:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.622021 8716 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:30:20.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.631736 8715 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 3 01:30:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.633527 8715 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.634194 8716 flags.cc:406] Enabled experimental flag: --ipki_ca_key_size=1024 01:30:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.634387 8716 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:30:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.634554 8716 flags.cc:406] Enabled experimental flag: --tsk_num_rsa_bits=512 01:30:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.634788 8716 flags.cc:406] Enabled experimental flag: --rpc_reuseport=true 01:30:20.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.646971 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 3 01:30:20.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.648409 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.661973 8716 master_main.cc:73] Master server non-default flags: 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/wal 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_ca_key_size=1024 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --master_addresses=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tsk_num_rsa_bits=512 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.188:45488 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_reuseport=true 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data/info.pb 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.188 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/logs 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Master server version: 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:30:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:30:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.678606 8716 master_main.cc:80] Initializing master server... 01:30:20.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.681346 8716 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 189878us 01:30:20.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.682375 8716 fs_manager.cc:263] Metadata directory not provided 01:30:20.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.682535 8716 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:30:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.682914 8716 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:30:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.683054 8716 server_base.cc:434] Attempting to create new FS layout instead 01:30:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.688216 8716 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data/instance: 01:30:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "41aeaf12b67b455987c502634d7d14d0" 01:30:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:20 on quasar-aecznm-1.vpc.cloudera.com" 01:30:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.689116 8716 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/wal/instance: 01:30:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "41aeaf12b67b455987c502634d7d14d0" 01:30:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:20 on quasar-aecznm-1.vpc.cloudera.com" 01:30:20.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.706570 8716 fs_manager.cc:503] Time spent creating directory manager: real 0.017s user 0.008s sys 0.008s 01:30:20.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.708144 8716 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:30:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.709874 8716 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:30:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.719573 8716 fs_manager.cc:419] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.719748 8716 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/wal 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "41aeaf12b67b455987c502634d7d14d0" 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:20 on quasar-aecznm-1.vpc.cloudera.com" 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.720122 8716 fs_report.cc:352] FS layout report 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/wal 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/wal 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data/data 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:30:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:30:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:30:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:30:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:30:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:30:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:30:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:30:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.788848 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 4 01:30:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.790591 8651 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.810008 8715 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 4 01:30:20.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:20.811944 8715 consensus_peers.cc:599] Error getting permanent uuid from config peer 127.8.97.188:45488: Network error: Client connection negotiation failed: client connection to 127.8.97.188:45488: connect: Connection refused (error 111) 01:30:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.907528 8716 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:30:20.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:20.910614 8716 master_main.cc:83] Starting Master server... 01:30:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.033696 8716 rpc_server.cc:205] RPC server started. Bound to: 127.8.97.188:45488 01:30:21.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.035027 8716 webserver.cc:175] Starting webserver on 127.8.97.188:0 01:30:21.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.035166 8716 webserver.cc:186] Document root disabled 01:30:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.037889 8716 webserver.cc:313] Webserver started. Bound to: http://127.8.97.188:41305/ 01:30:21.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.039021 8716 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/master-2/data/info.pb 01:30:21.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.044916 8780 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed 01:30:21.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.045831 8582 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master as pid 8716 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.046990 8582 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/wal 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.129:0 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.8.97.129 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.129 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/logs 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data/info.pb 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:30:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:30:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.055923 8651 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 5 01:30:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.066340 8780 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } has no permanent_uuid. Determining permanent_uuid... 01:30:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.089254 8651 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540: Bootstrap starting. 01:30:21.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.093482 8651 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540: No blocks or log segments found. Creating new log. 01:30:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.095317 8780 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } has no permanent_uuid. Determining permanent_uuid... 01:30:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.097158 8651 log.cc:525] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540: Log is configured to *not* fsync() on all Append() calls 01:30:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.110195 8780 sys_catalog.cc:304] member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } has no permanent_uuid. Determining permanent_uuid... 01:30:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.114388 8651 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540: No bootstrap required, opened a new log 01:30:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.115365 8715 consensus_peers.cc:609] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } attempt: 5 01:30:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.134264 8780 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: Bootstrap starting. 01:30:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.142662 8715 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: Bootstrap starting. 01:30:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.148058 8780 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: No blocks or log segments found. Creating new log. 01:30:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.147452 8651 raft_consensus.cc:340] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.148370 8651 raft_consensus.cc:366] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:30:21.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.149233 8651 raft_consensus.cc:705] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4ef7405012c144928614dbf254037540, State: Initialized, Role: FOLLOWER 01:30:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.150431 8651 consensus_queue.cc:229] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.153404 8715 tablet_bootstrap.cc:592] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: No blocks or log segments found. Creating new log. 01:30:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.155239 8715 log.cc:525] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: Log is configured to *not* fsync() on all Append() calls 01:30:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.158357 8780 log.cc:525] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: Log is configured to *not* fsync() on all Append() calls 01:30:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.155398 8651 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.159799 8651 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.161535 8651 sys_catalog.cc:424] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: configured and running, proceeding with master startup. 01:30:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.166293 8715 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: No bootstrap required, opened a new log 01:30:21.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.191799 8780 tablet_bootstrap.cc:438] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: No bootstrap required, opened a new log 01:30:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.208400 8715 raft_consensus.cc:340] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.209206 8715 raft_consensus.cc:366] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:30:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.213874 8715 raft_consensus.cc:705] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 28d5e62cff544540986a404b148f2a7f, State: Initialized, Role: FOLLOWER 01:30:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.214797 8715 consensus_queue.cc:229] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.224217 8780 raft_consensus.cc:340] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.227180 8780 raft_consensus.cc:366] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:30:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.227855 8780 raft_consensus.cc:705] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 41aeaf12b67b455987c502634d7d14d0, State: Initialized, Role: FOLLOWER 01:30:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.230142 8780 consensus_queue.cc:229] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.228704 8715 sys_catalog.cc:337] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.232969 8715 sys_catalog.cc:340] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.234453 8792 sys_catalog.cc:337] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.235790 8792 sys_catalog.cc:340] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.240145 8793 sys_catalog.cc:337] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.240919 8793 sys_catalog.cc:340] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.248554 8792 raft_consensus.cc:472] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:30:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.249131 8792 raft_consensus.cc:494] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.253998 8792 leader_election.cc:296] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4ef7405012c144928614dbf254037540 (127.8.97.190:34778), 41aeaf12b67b455987c502634d7d14d0 (127.8.97.188:45488) 01:30:21.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.159620 8788 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.333748 8788 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.334367 8788 raft_consensus.cc:472] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:30:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.334887 8788 raft_consensus.cc:494] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.341493 8788 leader_election.cc:296] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 28d5e62cff544540986a404b148f2a7f (127.8.97.189:35516), 41aeaf12b67b455987c502634d7d14d0 (127.8.97.188:45488) 01:30:21.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.354579 8607 tablet.cc:1783] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:30:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:30:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.349460 8781 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:30:21.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.355926 8781 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:30:21.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.392995 8781 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:30:21.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.395861 8781 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.97.129 01:30:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.235046 8715 sys_catalog.cc:424] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: configured and running, proceeding with master startup. 01:30:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.241369 8780 sys_catalog.cc:337] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: SysCatalogTable state changed. Reason: Started TabletReplica. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.402947 8780 sys_catalog.cc:340] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.404395 8780 sys_catalog.cc:424] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: configured and running, proceeding with master startup. 01:30:21.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.404624 8756 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ef7405012c144928614dbf254037540" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "41aeaf12b67b455987c502634d7d14d0" is_pre_election: true 01:30:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.404616 8757 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "28d5e62cff544540986a404b148f2a7f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "41aeaf12b67b455987c502634d7d14d0" is_pre_election: true 01:30:21.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.407924 8756 raft_consensus.cc:2346] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4ef7405012c144928614dbf254037540 in term 0. 01:30:21.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.408687 8757 raft_consensus.cc:2321] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 0 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 28d5e62cff544540986a404b148f2a7f for term 1 because replica is already servicing an update from a current leader or another vote. 01:30:21.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.411020 8601 leader_election.cc:310] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 41aeaf12b67b455987c502634d7d14d0, 4ef7405012c144928614dbf254037540; no voters: 01:30:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.412035 8788 raft_consensus.cc:2640] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Leader pre-election won for term 1 01:30:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.412309 8788 raft_consensus.cc:472] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:30:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.412557 8788 raft_consensus.cc:2886] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 0 FOLLOWER]: Advancing to term 1 01:30:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.420233 8716 master_main.cc:86] Master server successfully started. 01:30:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.416357 8736 tablet.cc:1783] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:30:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.419589 8788 raft_consensus.cc:494] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.430652 8788 leader_election.cc:296] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [CANDIDATE]: Term 1 election: Requested vote from peers 28d5e62cff544540986a404b148f2a7f (127.8.97.189:35516), 41aeaf12b67b455987c502634d7d14d0 (127.8.97.188:45488) 01:30:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.430662 8691 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ef7405012c144928614dbf254037540" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28d5e62cff544540986a404b148f2a7f" 01:30:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.431257 8757 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ef7405012c144928614dbf254037540" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "41aeaf12b67b455987c502634d7d14d0" 01:30:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.431509 8691 raft_consensus.cc:2886] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 0 FOLLOWER]: Advancing to term 1 01:30:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.431895 8757 raft_consensus.cc:2886] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 0 FOLLOWER]: Advancing to term 1 01:30:21.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.439186 8757 raft_consensus.cc:2346] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4ef7405012c144928614dbf254037540 in term 1. 01:30:21.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.439496 8691 raft_consensus.cc:2346] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4ef7405012c144928614dbf254037540 in term 1. 01:30:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.440446 8601 leader_election.cc:310] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 41aeaf12b67b455987c502634d7d14d0, 4ef7405012c144928614dbf254037540; no voters: 01:30:21.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.442122 8788 raft_consensus.cc:2640] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 1 FOLLOWER]: Leader election won for term 1 01:30:21.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.442533 8788 raft_consensus.cc:667] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 1 LEADER]: Becoming Leader. State: Replica: 4ef7405012c144928614dbf254037540, State: Running, Role: LEADER 01:30:21.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.443205 8788 consensus_queue.cc:206] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } 01:30:21.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.445664 8796 catalog_manager.cc:1104] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:30:21.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.447844 8796 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:30:21.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.255569 8628 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "28d5e62cff544540986a404b148f2a7f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4ef7405012c144928614dbf254037540" is_pre_election: true 01:30:21.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.453172 8791 catalog_manager.cc:529] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting 01:30:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.455605 8590 master_main.cc:86] Master server successfully started. 01:30:21.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.465481 8628 raft_consensus.cc:2307] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [term 1 LEADER]: Leader pre-election vote request: Denying vote to candidate 28d5e62cff544540986a404b148f2a7f for term 1 because replica is either leader or believes a valid leader to be alive. 01:30:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.469429 8794 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ef7405012c144928614dbf254037540. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.470355 8794 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: This master's current role is: LEADER 01:30:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.470558 8662 leader_election.cc:310] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28d5e62cff544540986a404b148f2a7f; no voters: 41aeaf12b67b455987c502634d7d14d0, 4ef7405012c144928614dbf254037540 01:30:21.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.471786 8792 raft_consensus.cc:2588] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority 01:30:21.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.479001 8801 catalog_manager.cc:1046] Loading table and tablet metadata into memory... 01:30:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.265257 8671 tablet.cc:1783] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:30:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.497237 8801 catalog_manager.cc:1057] Initializing Kudu internal certificate authority... 01:30:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.505208 8781 tablet_server_main.cc:78] Tablet server non-default flags: 01:30:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data 01:30:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/wal 01:30:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.129:0 01:30:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data/info.pb 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.129 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.8.97.129 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/logs 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:30:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:30:21.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:30:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.343013 8692 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ef7405012c144928614dbf254037540" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28d5e62cff544540986a404b148f2a7f" is_pre_election: true 01:30:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.545509 8692 raft_consensus.cc:2259] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 4ef7405012c144928614dbf254037540 in term 1. Re-sending same reply. 01:30:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.550052 8652 master_main.cc:86] Master server successfully started. 01:30:21.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.558770 8781 tablet_server_main.cc:85] Initializing tablet server... 01:30:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.561456 8795 catalog_manager.cc:1104] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: acquiring CA information for follower catalog manager: Not found: root CA entry not found 01:30:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:21.561779 8795 catalog_manager.cc:587] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry 01:30:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.564438 8781 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 190378us 01:30:21.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.565467 8781 fs_manager.cc:263] Metadata directory not provided 01:30:21.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.565951 8781 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:30:21.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.566568 8781 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:30:21.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.566921 8781 server_base.cc:434] Attempting to create new FS layout instead 01:30:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.574831 8781 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data/instance: 01:30:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b18778d645364e8bb6fac43fb32aa104" 01:30:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:21 on quasar-aecznm-1.vpc.cloudera.com" 01:30:21.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.576862 8781 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/wal/instance: 01:30:21.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b18778d645364e8bb6fac43fb32aa104" 01:30:21.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:21 on quasar-aecznm-1.vpc.cloudera.com" 01:30:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.610034 8781 fs_manager.cc:503] Time spent creating directory manager: real 0.032s user 0.020s sys 0.005s 01:30:21.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.613919 8781 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:30:21.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.614624 8757 raft_consensus.cc:1181] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [term 1 FOLLOWER]: Refusing update from remote peer 4ef7405012c144928614dbf254037540: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:30:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.614897 8692 raft_consensus.cc:1181] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [term 1 FOLLOWER]: Refusing update from remote peer 4ef7405012c144928614dbf254037540: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:30:21.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.616814 8781 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:30:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.621219 8788 consensus_queue.cc:984] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [LEADER]: Connected to new peer: Peer: permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:30:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.622000 8794 consensus_queue.cc:984] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [LEADER]: Connected to new peer: Peer: permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:30:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.637153 8781 fs_manager.cc:419] Time spent opening block manager: real 0.008s user 0.002s sys 0.004s 01:30:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.637653 8781 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/wal 01:30:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "b18778d645364e8bb6fac43fb32aa104" 01:30:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:21 on quasar-aecznm-1.vpc.cloudera.com" 01:30:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.638417 8781 fs_report.cc:352] FS layout report 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/wal 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/wal 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data/data 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:30:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:30:21.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.654942 8792 sys_catalog.cc:337] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ef7405012c144928614dbf254037540. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.655792 8792 sys_catalog.cc:340] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.668644 8793 sys_catalog.cc:337] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ef7405012c144928614dbf254037540. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.669600 8793 sys_catalog.cc:340] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.672067 8810 mvcc.cc:203] Tried to move safe_time back from 6351129074114658304 to 6351129073440092160. Current Snapshot: MvccSnapshot[committed={T|T < 6351129074114658304}] 01:30:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.681279 8792 sys_catalog.cc:337] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.682175 8792 sys_catalog.cc:340] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.687387 8801 catalog_manager.cc:941] Generated new certificate authority record 01:30:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.690445 8801 catalog_manager.cc:1066] Loading token signing keys... 01:30:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.701004 8788 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.701130 8794 sys_catalog.cc:337] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.701882 8788 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: This master's current role is: LEADER 01:30:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.702538 8794 sys_catalog.cc:340] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540 [sys.catalog]: This master's current role is: LEADER 01:30:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.715306 8801 catalog_manager.cc:4095] T 00000000000000000000000000000000 P 4ef7405012c144928614dbf254037540: Generated new TSK 0 01:30:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.729766 8793 sys_catalog.cc:337] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4ef7405012c144928614dbf254037540" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ef7405012c144928614dbf254037540" member_type: VOTER last_known_addr { host: "127.8.97.190" port: 34778 } } peers { permanent_uuid: "28d5e62cff544540986a404b148f2a7f" member_type: VOTER last_known_addr { host: "127.8.97.189" port: 35516 } } peers { permanent_uuid: "41aeaf12b67b455987c502634d7d14d0" member_type: VOTER last_known_addr { host: "127.8.97.188" port: 45488 } } } 01:30:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.730636 8793 sys_catalog.cc:340] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0 [sys.catalog]: This master's current role is: FOLLOWER 01:30:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.927184 8781 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:30:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.932399 8781 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:30:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:21.935890 8781 tablet_server_main.cc:90] Starting tablet server... 01:30:22.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.401206 8781 rpc_server.cc:205] RPC server started. Bound to: 127.8.97.129:59546 01:30:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.403321 8781 webserver.cc:175] Starting webserver on 127.8.97.129:0 01:30:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.403676 8781 webserver.cc:186] Document root disabled 01:30:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.411850 8781 webserver.cc:313] Webserver started. Bound to: http://127.8.97.129:60937/ 01:30:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.413427 8781 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-0/data/info.pb 01:30:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.422685 8582 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 8781 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.424154 8582 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/wal 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.130:0 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.8.97.130 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.130 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/logs 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data/info.pb 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:30:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:30:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:30:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.459450 8796 catalog_manager.cc:1102] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: acquiring CA information for follower catalog manager: success 01:30:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.470614 8796 catalog_manager.cc:1130] T 00000000000000000000000000000000 P 41aeaf12b67b455987c502634d7d14d0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 01:30:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.487145 8781 tablet_server_main.cc:93] Tablet server successfully started. 01:30:22.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.511675 8940 heartbeater.cc:345] Connected to a master server at 127.8.97.190:34778 01:30:22.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.512063 8940 heartbeater.cc:426] Registering TS with master... 01:30:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.514088 8940 heartbeater.cc:475] Master 127.8.97.190:34778 requested a full tablet report, sending... 01:30:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.517377 8618 ts_manager.cc:98] Registered new tserver with Master: b18778d645364e8bb6fac43fb32aa104 (127.8.97.129:59546) 01:30:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.520243 8935 heartbeater.cc:345] Connected to a master server at 127.8.97.188:45488 01:30:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.520517 8935 heartbeater.cc:426] Registering TS with master... 01:30:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.520609 8618 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.8.97.129:37420 01:30:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.521684 8935 heartbeater.cc:475] Master 127.8.97.188:45488 requested a full tablet report, sending... 01:30:22.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.525904 8747 ts_manager.cc:98] Registered new tserver with Master: b18778d645364e8bb6fac43fb32aa104 (127.8.97.129:59546) 01:30:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.513175 8938 heartbeater.cc:345] Connected to a master server at 127.8.97.189:35516 01:30:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.542362 8938 heartbeater.cc:426] Registering TS with master... 01:30:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.542888 8938 heartbeater.cc:475] Master 127.8.97.189:35516 requested a full tablet report, sending... 01:30:22.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.546749 8682 ts_manager.cc:98] Registered new tserver with Master: b18778d645364e8bb6fac43fb32aa104 (127.8.97.129:59546) 01:30:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.569684 8795 catalog_manager.cc:1102] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: acquiring CA information for follower catalog manager: success 01:30:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.576839 8795 catalog_manager.cc:1130] T 00000000000000000000000000000000 P 28d5e62cff544540986a404b148f2a7f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0 01:30:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:30:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:22.769412 8939 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:30:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:22.770515 8939 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:30:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:22.789892 8939 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:30:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:22.790648 8939 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.97.130 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.841660 8939 tablet_server_main.cc:78] Tablet server non-default flags: 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/wal 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.130:0 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data/info.pb 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.130 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.8.97.130 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/logs 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:30:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:30:22.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.874346 8939 tablet_server_main.cc:85] Initializing tablet server... 01:30:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.878866 8939 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 190878us 01:30:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.879950 8939 fs_manager.cc:263] Metadata directory not provided 01:30:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.880383 8939 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:30:22.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.881078 8939 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:30:22.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.881476 8939 server_base.cc:434] Attempting to create new FS layout instead 01:30:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.888187 8939 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data/instance: 01:30:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "49c72263ff5c41848addbf216e68983f" 01:30:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:22 on quasar-aecznm-1.vpc.cloudera.com" 01:30:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.889959 8939 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/wal/instance: 01:30:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "49c72263ff5c41848addbf216e68983f" 01:30:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:22 on quasar-aecznm-1.vpc.cloudera.com" 01:30:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.915761 8939 fs_manager.cc:503] Time spent creating directory manager: real 0.025s user 0.012s sys 0.010s 01:30:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.918320 8939 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:30:22.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.919401 8939 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:30:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.938344 8939 fs_manager.cc:419] Time spent opening block manager: real 0.007s user 0.001s sys 0.005s 01:30:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.938824 8939 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/wal 01:30:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "49c72263ff5c41848addbf216e68983f" 01:30:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:22 on quasar-aecznm-1.vpc.cloudera.com" 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:22.939611 8939 fs_report.cc:352] FS layout report 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/wal 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/wal 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data/data 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:30:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:30:23.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.198169 8939 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:30:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.207319 8939 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:30:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.213680 8939 tablet_server_main.cc:90] Starting tablet server... 01:30:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.525616 8940 heartbeater.cc:467] Master 127.8.97.190:34778 was elected leader, sending a full tablet report... 01:30:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.933410 8939 rpc_server.cc:205] RPC server started. Bound to: 127.8.97.130:47565 01:30:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.935328 8939 webserver.cc:175] Starting webserver on 127.8.97.130:0 01:30:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.935452 8939 webserver.cc:186] Document root disabled 01:30:23.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.942332 8939 webserver.cc:313] Webserver started. Bound to: http://127.8.97.130:46284/ 01:30:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.943915 8939 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-1/data/info.pb 01:30:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.953096 8582 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 8939 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.954479 8582 external_mini_cluster.cc:933] Running /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/wal 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --block_manager=log 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.131:0 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.8.97.131 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.131 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/logs 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data/info.pb 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags 01:30:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags with env {} 01:30:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:23.998471 8939 tablet_server_main.cc:93] Tablet server successfully started. 01:30:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.050593 9066 heartbeater.cc:345] Connected to a master server at 127.8.97.188:45488 01:30:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.051045 9066 heartbeater.cc:426] Registering TS with master... 01:30:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.051887 9066 heartbeater.cc:475] Master 127.8.97.188:45488 requested a full tablet report, sending... 01:30:24.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.055757 8747 ts_manager.cc:98] Registered new tserver with Master: 49c72263ff5c41848addbf216e68983f (127.8.97.130:47565) 01:30:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.062335 9067 heartbeater.cc:345] Connected to a master server at 127.8.97.189:35516 01:30:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.062623 9067 heartbeater.cc:426] Registering TS with master... 01:30:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.063189 9067 heartbeater.cc:475] Master 127.8.97.189:35516 requested a full tablet report, sending... 01:30:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.066491 8682 ts_manager.cc:98] Registered new tserver with Master: 49c72263ff5c41848addbf216e68983f (127.8.97.130:47565) 01:30:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.069361 9071 heartbeater.cc:345] Connected to a master server at 127.8.97.190:34778 01:30:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.069633 9071 heartbeater.cc:426] Registering TS with master... 01:30:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.070210 9071 heartbeater.cc:475] Master 127.8.97.190:34778 requested a full tablet report, sending... 01:30:24.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.072563 8618 ts_manager.cc:98] Registered new tserver with Master: 49c72263ff5c41848addbf216e68983f (127.8.97.130:47565) 01:30:24.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.074615 8618 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.8.97.130:37814 01:30:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) WARNING: Logging before InitGoogleLogging() is written to STDERR 01:30:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:24.370910 9069 flags.cc:406] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true 01:30:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:24.371620 9069 flags.cc:406] Enabled unsafe flag: --never_fsync=true 01:30:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:24.391651 9069 flags.cc:406] Enabled experimental flag: --ipki_server_key_size=1024 01:30:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:24.392513 9069 flags.cc:406] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.97.131 01:30:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.432330 9069 tablet_server_main.cc:78] Tablet server non-default flags: 01:30:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_data_dirs=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --fs_wal_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/wal 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --ipki_server_key_size=1024 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_bind_addresses=127.8.97.131:0 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --rpc_server_allow_ephemeral_ports=true 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --metrics_log_interval_ms=1000 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_format=pb 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --server_dump_info_path=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data/info.pb 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_interface=127.8.97.131 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --webserver_port=0 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --tserver_master_addrs=127.8.97.190:34778,127.8.97.189:35516,127.8.97.188:45488 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --never_fsync=true 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --redact=none 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_experimental_flags=true 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --unlock_unsafe_flags=true 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --enable_minidumps=false 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --local_ip_for_outbound_sockets=127.8.97.131 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --log_dir=/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/logs 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logbuflevel=-1 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) --logtostderr=true 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Tablet server version: 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) kudu 1.10.0-SNAPSHOT 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) revision 2be008763cda5eb28c0dfa8863e396575fce91ab 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build type FASTDEBUG 01:30:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) built by jenkins at 18 Feb 2019 22:41:05 PST on quasar-aecznm-1.vpc.cloudera.com 01:30:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) build id 263466 01:30:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) TSAN enabled 01:30:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.461788 9069 tablet_server_main.cc:85] Initializing tablet server... 01:30:24.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.465651 9069 system_ntp.cc:199] NTP initialized. Skew: 500ppm Current error: 191878us 01:30:24.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.466668 9069 fs_manager.cc:263] Metadata directory not provided 01:30:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.467068 9069 fs_manager.cc:269] Using write-ahead log directory (fs_wal_dir) as metadata directory 01:30:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.467640 9069 server_base.cc:433] Could not load existing FS layout: Not found: could not find a healthy instance file 01:30:24.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.468022 9069 server_base.cc:434] Attempting to create new FS layout instead 01:30:24.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.474694 9069 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data/instance: 01:30:24.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "6928ede89be54bbab71eedab94e61d67" 01:30:24.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:24 on quasar-aecznm-1.vpc.cloudera.com" 01:30:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.476287 9069 fs_manager.cc:602] Generated new instance metadata in path /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/wal/instance: 01:30:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "6928ede89be54bbab71eedab94e61d67" 01:30:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:24 on quasar-aecznm-1.vpc.cloudera.com" 01:30:24.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.495146 9069 fs_manager.cc:503] Time spent creating directory manager: real 0.018s user 0.015s sys 0.001s 01:30:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.497143 9069 env_posix.cc:1676] Not raising this process' open files per process limit of 32768; it is already as high as it can go 01:30:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.497910 9069 file_cache.cc:466] Constructed file cache lbm with capacity 13107 01:30:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.507863 9069 fs_manager.cc:419] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s 01:30:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.508059 9069 fs_manager.cc:436] Opened local filesystem: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data,/data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/wal 01:30:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) uuid: "6928ede89be54bbab71eedab94e61d67" 01:30:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) format_stamp: "Formatted at 2019-02-19 09:30:24 on quasar-aecznm-1.vpc.cloudera.com" 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.508519 9069 fs_report.cc:352] FS layout report 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) -------------------- 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) wal directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/wal 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) metadata directory: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/wal 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) 1 data directories: /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data/data 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live blocks: 0 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes: 0 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total live bytes (after alignment): 0 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total number of LBM containers: 0 (0 full) 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for missing blocks 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Did not check for orphaned blocks 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM containers with extra space: 0 (0 repaired) 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total full LBM container extra space in bytes: 0 (0 repaired) 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total incomplete LBM containers: 0 (0 repaired) 01:30:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) Total LBM partial records: 0 (0 repaired) 01:30:24.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.717689 9069 env_posix.cc:1671] Raising this process' running threads per effective uid limit from 32768 to 65536 01:30:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.722544 9069 ts_tablet_manager.cc:356] Loaded tablet metadata (0 live tablets) 01:30:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.724406 9069 tablet_server_main.cc:90] Starting tablet server... 01:30:24.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.870240 9069 rpc_server.cc:205] RPC server started. Bound to: 127.8.97.131:52775 01:30:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.871963 9069 webserver.cc:175] Starting webserver on 127.8.97.131:0 01:30:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.872118 9069 webserver.cc:186] Document root disabled 01:30:24.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.874379 9069 webserver.cc:313] Webserver started. Bound to: http://127.8.97.131:53461/ 01:30:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.875492 9069 server_base.cc:608] Dumped server information to /data/somelongdirectorytoavoidrpathissues/src/kudutest/mini-kudu-cluster8585393289005357278/ts-2/data/info.pb 01:30:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.881614 8582 external_mini_cluster.cc:995] Started /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver as pid 9069 01:30:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.898558 9069 tablet_server_main.cc:93] Tablet server successfully started. 01:30:24.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.979622 9199 heartbeater.cc:345] Connected to a master server at 127.8.97.190:34778 01:30:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.980180 9199 heartbeater.cc:426] Registering TS with master... 01:30:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.981431 9199 heartbeater.cc:475] Master 127.8.97.190:34778 requested a full tablet report, sending... 01:30:24.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.983997 8618 ts_manager.cc:98] Registered new tserver with Master: 6928ede89be54bbab71eedab94e61d67 (127.8.97.131:52775) 01:30:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.985492 8618 master_service.cc:269] Signed X509 certificate for tserver {username='jenkins'} at 127.8.97.131:43660 01:30:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.987948 9197 heartbeater.cc:345] Connected to a master server at 127.8.97.188:45488 01:30:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.988299 9197 heartbeater.cc:426] Registering TS with master... 01:30:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.989588 9197 heartbeater.cc:475] Master 127.8.97.188:45488 requested a full tablet report, sending... 01:30:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.992290 9198 heartbeater.cc:345] Connected to a master server at 127.8.97.189:35516 01:30:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.992602 9198 heartbeater.cc:426] Registering TS with master... 01:30:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.993085 8747 ts_manager.cc:98] Registered new tserver with Master: 6928ede89be54bbab71eedab94e61d67 (127.8.97.131:52775) 01:30:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.993126 9198 heartbeater.cc:475] Master 127.8.97.189:35516 requested a full tablet report, sending... 01:30:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.996871 8682 ts_manager.cc:98] Registered new tserver with Master: 6928ede89be54bbab71eedab94e61d67 (127.8.97.131:52775) 01:30:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:24.999370 8582 external_mini_cluster.cc:623] 3 TS(s) registered with all masters 01:30:24.999 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: 01:30:25.000 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: get_masters { } 01:30:25.001 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: get_masters { masters { id { type: MASTER index: 0 } bound_rpc_address { host: "127.8.97.190" port: 34778 } } masters { id { type: MASTER index: 1 } bound_rpc_address { host: "127.8.97.189" port: 35516 } } masters { id { type: MASTER index: 2 } bound_rpc_address { host: "127.8.97.188" port: 45488 } } } 01:30:25.001 [DEBUG - Test worker] (MiniKuduCluster.java:155) Request: get_tservers { } 01:30:25.002 [DEBUG - Test worker] (MiniKuduCluster.java:165) Response: get_tservers { tservers { id { type: TSERVER index: 0 } bound_rpc_address { host: "127.8.97.129" port: 59546 } } tservers { id { type: TSERVER index: 1 } bound_rpc_address { host: "127.8.97.130" port: 47565 } } tservers { id { type: TSERVER index: 2 } bound_rpc_address { host: "127.8.97.131" port: 52775 } } } 01:30:25.002 [INFO - Test worker] (KuduTestHarness.java:142) Creating a new Kudu client... 01:30:25.005 [DEBUG - Test worker] (SecurityUtil.java:123) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 01:30:25.008 [DEBUG - New I/O worker #2170] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.8.97.189 01:30:25.011 [DEBUG - New I/O worker #2171] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.8.97.188 01:30:25.011 [DEBUG - New I/O worker #2172] (Negotiator.java:460) SASL mechanism PLAIN chosen for peer 127.8.97.190 01:30:25.017 [DEBUG - New I/O worker #2170] (Negotiator.java:764) Authenticated connection [id: 0xd3100727, /127.8.97.189:38641 => /127.8.97.189:35516] using SASL/PLAIN 01:30:25.020 [DEBUG - New I/O worker #2171] (Negotiator.java:764) Authenticated connection [id: 0xb0781feb, /127.8.97.188:45606 => /127.8.97.188:45488] using SASL/PLAIN 01:30:25.024 [DEBUG - New I/O worker #2172] (Negotiator.java:764) Authenticated connection [id: 0xa1df81ce, /127.8.97.190:52661 => /127.8.97.190:34778] using SASL/PLAIN 01:30:25.027 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:2118) Learned about tablet Kudu Master for table 'Kudu Master' with partition [, ) 01:30:25.028 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, DeadlineTracker(timeout=50000, elapsed=21), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.97.189:35516, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.97.188:45488, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.97.190:34778, [17ms] Sub RPC ConnectToMaster: received response from server master-127.8.97.188:45488: OK, [17ms] Sub RPC ConnectToMaster: received response from server master-127.8.97.189:35516: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.8.97.190:34778: OK, deferred=null) after lookup 01:30:25.028 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=retry RPC@1154769582 returned Deferred@1089208044(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after retry RPC@1154769582), errback=(continuation of Deferred@99182741 after retry RPC@1154769582)), so the following Deferred is getting paused: Deferred@99182741(state=PAUSED, result=Deferred@1089208044, callback=org.apache.kudu.client.AsyncKuduClient$1@5380b2c1 -> org.apache.kudu.client.AsyncKuduClient$2@34266b0d -> wakeup thread Time-limited test, errback=passthrough -> passthrough -> wakeup thread Time-limited test) 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.029520 8618 catalog_manager.cc:1386] Servicing CreateTable request from {username='jenkins'} at 127.8.97.190:52661: 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "testMultipleSessions" 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) schema { 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: true 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column1_i" 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column2_i" 01:30:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: INT32 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column3_s" 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: STRING 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: true 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) encoding: DICT_ENCODING 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) compression: LZ4 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 4096 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "column4_b" 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) type: BOOL 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_key: false 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) is_nullable: false 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) cfile_block_size: 0 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) partition_schema { 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) range_schema { 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) columns { 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) name: "key" 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) } 01:30:25.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:25.042419 8618 catalog_manager.cc:1537] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table testMultipleSessions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers. 01:30:25.063 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$1@5380b2c1@1400943297 returned Deferred@732295163(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduClient$6@65c697da -> (continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$1@5380b2c1@1400943297), errback=passthrough -> (continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$1@5380b2c1@1400943297)), so the following Deferred is getting paused: Deferred@99182741(state=PAUSED, result=Deferred@732295163, callback=org.apache.kudu.client.AsyncKuduClient$2@34266b0d -> wakeup thread Time-limited test, errback=passthrough -> wakeup thread Time-limited test) 01:30:25.068 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:761) Opened table a934496b2cea45339a5f4d9509ca6fe2 01:30:25.068 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005)), so the following Deferred is getting paused: Deferred@99182741(state=PAUSED, result=Deferred@864433945, callback=wakeup thread Time-limited test, errback=wakeup thread Time-limited test) 01:30:25.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.078894 9071 heartbeater.cc:467] Master 127.8.97.190:34778 was elected leader, sending a full tablet report... 01:30:25.100 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471)), so the following Deferred is getting paused: Deferred@815900468(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.113485 8872 tablet_service.cc:870] Processing CreateTablet for tablet 80798175d30f473995c98f40d358813b (table=testMultipleSessions [id=a934496b2cea45339a5f4d9509ca6fe2]), partition=RANGE (key) PARTITION UNBOUNDED 01:30:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.114534 8872 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 80798175d30f473995c98f40d358813b. 1 dirs total, 0 dirs full, 0 dirs failed 01:30:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.115192 9134 tablet_service.cc:870] Processing CreateTablet for tablet 80798175d30f473995c98f40d358813b (table=testMultipleSessions [id=a934496b2cea45339a5f4d9509ca6fe2]), partition=RANGE (key) PARTITION UNBOUNDED 01:30:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.116786 9134 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 80798175d30f473995c98f40d358813b. 1 dirs total, 0 dirs full, 0 dirs failed 01:30:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.131580 8872 ts_tablet_manager.cc:1175] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Registered tablet (data state: TABLET_DATA_READY) 01:30:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.134083 9238 ts_tablet_manager.cc:1030] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Bootstrapping tablet 01:30:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.135236 9238 tablet_bootstrap.cc:438] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Bootstrap starting. 01:30:25.139 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389)), so the following Deferred is getting paused: Deferred@1672762218(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.139637 9238 tablet_bootstrap.cc:592] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: No blocks or log segments found. Creating new log. 01:30:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.141261 9003 tablet_service.cc:870] Processing CreateTablet for tablet 80798175d30f473995c98f40d358813b (table=testMultipleSessions [id=a934496b2cea45339a5f4d9509ca6fe2]), partition=RANGE (key) PARTITION UNBOUNDED 01:30:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.141639 9238 log.cc:525] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Log is configured to *not* fsync() on all Append() calls 01:30:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.142561 9003 data_dirs.cc:938] Could only allocate 1 dirs of requested 3 for tablet 80798175d30f473995c98f40d358813b. 1 dirs total, 0 dirs full, 0 dirs failed 01:30:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.146036 9238 tablet_bootstrap.cc:438] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: No bootstrap required, opened a new log 01:30:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.146762 9238 ts_tablet_manager.cc:1047] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Time spent bootstrapping tablet: real 0.012s user 0.007s sys 0.002s 01:30:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.148180 9134 ts_tablet_manager.cc:1175] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Registered tablet (data state: TABLET_DATA_READY) 01:30:25.148 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767)), so the following Deferred is getting paused: Deferred@281230720(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.152657 9240 ts_tablet_manager.cc:1030] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Bootstrapping tablet 01:30:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.153834 9240 tablet_bootstrap.cc:438] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Bootstrap starting. 01:30:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.159670 9240 tablet_bootstrap.cc:592] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: No blocks or log segments found. Creating new log. 01:30:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.160974 9003 ts_tablet_manager.cc:1175] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Registered tablet (data state: TABLET_DATA_READY) 01:30:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.162580 9240 log.cc:525] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Log is configured to *not* fsync() on all Append() calls 01:30:25.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.163092 9241 ts_tablet_manager.cc:1030] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Bootstrapping tablet 01:30:25.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.164106 9241 tablet_bootstrap.cc:438] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Bootstrap starting. 01:30:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.168236 9240 tablet_bootstrap.cc:438] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: No bootstrap required, opened a new log 01:30:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.168433 9241 tablet_bootstrap.cc:592] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: No blocks or log segments found. Creating new log. 01:30:25.169 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675)), so the following Deferred is getting paused: Deferred@1962253977(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.168792 9240 ts_tablet_manager.cc:1047] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Time spent bootstrapping tablet: real 0.016s user 0.007s sys 0.005s 01:30:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.170770 9241 log.cc:525] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Log is configured to *not* fsync() on all Append() calls 01:30:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.171756 9238 raft_consensus.cc:340] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.172900 9238 raft_consensus.cc:366] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:30:25.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.173377 9238 raft_consensus.cc:705] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b18778d645364e8bb6fac43fb32aa104, State: Initialized, Role: FOLLOWER 01:30:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.174594 9241 tablet_bootstrap.cc:438] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: No bootstrap required, opened a new log 01:30:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.175089 9241 ts_tablet_manager.cc:1047] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.004s 01:30:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.174437 9238 consensus_queue.cc:229] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.190 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999)), so the following Deferred is getting paused: Deferred@1704300268(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.191740 9238 ts_tablet_manager.cc:1075] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Time spent starting tablet: real 0.045s user 0.025s sys 0.006s 01:30:25.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.205585 9240 raft_consensus.cc:340] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.205518 9241 raft_consensus.cc:340] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending transactions. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.207128 9240 raft_consensus.cc:366] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:30:25.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.207476 9241 raft_consensus.cc:366] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely 01:30:25.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.207520 9240 raft_consensus.cc:705] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6928ede89be54bbab71eedab94e61d67, State: Initialized, Role: FOLLOWER 01:30:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.208182 9241 raft_consensus.cc:705] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 49c72263ff5c41848addbf216e68983f, State: Initialized, Role: FOLLOWER 01:30:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.208649 9240 consensus_queue.cc:229] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.209405 9241 consensus_queue.cc:229] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.218897 9199 heartbeater.cc:467] Master 127.8.97.190:34778 was elected leader, sending a full tablet report... 01:30:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.222479 9240 ts_tablet_manager.cc:1075] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Time spent starting tablet: real 0.053s user 0.025s sys 0.011s 01:30:25.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:25.228265 8943 tablet.cc:1783] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:30:25.232 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225)), so the following Deferred is getting paused: Deferred@1436373462(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.239101 9241 ts_tablet_manager.cc:1075] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Time spent starting tablet: real 0.064s user 0.026s sys 0.011s 01:30:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.243383 9245 raft_consensus.cc:472] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout) 01:30:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.243996 9245 raft_consensus.cc:494] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.249773 9245 leader_election.cc:296] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 49c72263ff5c41848addbf216e68983f (127.8.97.130:47565), b18778d645364e8bb6fac43fb32aa104 (127.8.97.129:59546) 01:30:25.270 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723)), so the following Deferred is getting paused: Deferred@1820055331(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.274024 9023 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "80798175d30f473995c98f40d358813b" candidate_uuid: "6928ede89be54bbab71eedab94e61d67" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49c72263ff5c41848addbf216e68983f" is_pre_election: true 01:30:25.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.275022 9023 raft_consensus.cc:2346] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6928ede89be54bbab71eedab94e61d67 in term 0. 01:30:25.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.274969 8892 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "80798175d30f473995c98f40d358813b" candidate_uuid: "6928ede89be54bbab71eedab94e61d67" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b18778d645364e8bb6fac43fb32aa104" is_pre_election: true 01:30:25.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.275867 8892 raft_consensus.cc:2346] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6928ede89be54bbab71eedab94e61d67 in term 0. 01:30:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.278201 9086 leader_election.cc:310] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6928ede89be54bbab71eedab94e61d67, b18778d645364e8bb6fac43fb32aa104; no voters: 01:30:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.279595 9245 raft_consensus.cc:2640] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Leader pre-election won for term 1 01:30:25.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.279987 9245 raft_consensus.cc:472] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout) 01:30:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.280320 9245 raft_consensus.cc:2886] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 0 FOLLOWER]: Advancing to term 1 01:30:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.288326 9245 raft_consensus.cc:494] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.292762 9023 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "80798175d30f473995c98f40d358813b" candidate_uuid: "6928ede89be54bbab71eedab94e61d67" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49c72263ff5c41848addbf216e68983f" 01:30:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.293416 9023 raft_consensus.cc:2886] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 0 FOLLOWER]: Advancing to term 1 01:30:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.293640 8892 tablet_service.cc:1122] Received RequestConsensusVote() RPC: tablet_id: "80798175d30f473995c98f40d358813b" candidate_uuid: "6928ede89be54bbab71eedab94e61d67" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b18778d645364e8bb6fac43fb32aa104" 01:30:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.294256 8892 raft_consensus.cc:2886] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 0 FOLLOWER]: Advancing to term 1 01:30:25.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.300503 9023 raft_consensus.cc:2346] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6928ede89be54bbab71eedab94e61d67 in term 1. 01:30:25.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.300621 8892 raft_consensus.cc:2346] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6928ede89be54bbab71eedab94e61d67 in term 1. 01:30:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.302525 9086 leader_election.cc:310] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6928ede89be54bbab71eedab94e61d67, b18778d645364e8bb6fac43fb32aa104; no voters: 01:30:25.349 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723) -> (continuation of Deferred@2114180736 after org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723) -> (continuation of Deferred@2114180736 after org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642)), so the following Deferred is getting paused: Deferred@2114180736(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.368 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@5d8cf2dc@1569518300 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723) -> (continuation of Deferred@2114180736 after org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642) -> (continuation of Deferred@1878844545 after org.apache.kudu.client.AsyncKuduClient$15@5d8cf2dc@1569518300), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723) -> (continuation of Deferred@2114180736 after org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642) -> (continuation of Deferred@1878844545 after org.apache.kudu.client.AsyncKuduClient$15@5d8cf2dc@1569518300)), so the following Deferred is getting paused: Deferred@1878844545(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:25.261881 9072 tablet.cc:1783] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:30:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.305380 9245 leader_election.cc:296] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [CANDIDATE]: Term 1 election: Requested vote from peers 49c72263ff5c41848addbf216e68983f (127.8.97.130:47565), b18778d645364e8bb6fac43fb32aa104 (127.8.97.129:59546) 01:30:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.418833 9245 raft_consensus.cc:2640] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 1 FOLLOWER]: Leader election won for term 1 01:30:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:25.418607 9086 outbound_call.cc:287] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.8.97.129:59546, user_credentials={real_user=jenkins}} blocked reactor thread for 116141us 01:30:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.419309 9245 raft_consensus.cc:667] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [term 1 LEADER]: Becoming Leader. State: Replica: 6928ede89be54bbab71eedab94e61d67, State: Running, Role: LEADER 01:30:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.420226 9245 consensus_queue.cc:206] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } } 01:30:25.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:25.388356 9201 tablet.cc:1783] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67: Can't schedule compaction. Clean time has not been advanced past its initial value. 01:30:25.449 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=org.apache.kudu.client.AsyncKuduClient$15@578f1e50@1468997200 returned Deferred@864433945(state=PENDING, result=null, callback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723) -> (continuation of Deferred@2114180736 after org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642) -> (continuation of Deferred@1878844545 after org.apache.kudu.client.AsyncKuduClient$15@5d8cf2dc@1569518300) -> (continuation of Deferred@928544247 after org.apache.kudu.client.AsyncKuduClient$15@578f1e50@1468997200), errback=(continuation of Deferred@99182741 after org.apache.kudu.client.AsyncKuduClient$2@34266b0d@874933005) -> (continuation of Deferred@815900468 after org.apache.kudu.client.AsyncKuduClient$15@34ff4c67@889146471) -> (continuation of Deferred@1672762218 after org.apache.kudu.client.AsyncKuduClient$15@61888ebd@1636339389) -> (continuation of Deferred@281230720 after org.apache.kudu.client.AsyncKuduClient$15@2de6a70f@770090767) -> (continuation of Deferred@1962253977 after org.apache.kudu.client.AsyncKuduClient$15@359bb443@899396675) -> (continuation of Deferred@1704300268 after org.apache.kudu.client.AsyncKuduClient$15@6c3391d7@1815318999) -> (continuation of Deferred@1436373462 after org.apache.kudu.client.AsyncKuduClient$15@d4407b1@222562225) -> (continuation of Deferred@1820055331 after org.apache.kudu.client.AsyncKuduClient$15@25d0df9b@634445723) -> (continuation of Deferred@2114180736 after org.apache.kudu.client.AsyncKuduClient$15@412f2c92@1093610642) -> (continuation of Deferred@1878844545 after org.apache.kudu.client.AsyncKuduClient$15@5d8cf2dc@1569518300) -> (continuation of Deferred@928544247 after org.apache.kudu.client.AsyncKuduClient$15@578f1e50@1468997200)), so the following Deferred is getting paused: Deferred@928544247(state=PAUSED, result=Deferred@864433945, callback=, errback=) 01:30:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.456049 8617 catalog_manager.cc:3791] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 reported cstate change: term changed from 0 to 1, leader changed from to 6928ede89be54bbab71eedab94e61d67 (127.8.97.131). New cstate: current_term: 1 leader_uuid: "6928ede89be54bbab71eedab94e61d67" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6928ede89be54bbab71eedab94e61d67" member_type: VOTER last_known_addr { host: "127.8.97.131" port: 52775 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 } health_report { overall_health: UNKNOWN } } } 01:30:25.573 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:2118) Learned about tablet 80798175d30f473995c98f40d358813b for table 'testMultipleSessions' with partition [, ) 01:30:25.574 [DEBUG - New I/O worker #2172] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=80798175d30f473995c98f40d358813b@[49c72263ff5c41848addbf216e68983f(127.8.97.130:47565),6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775)[L],b18778d645364e8bb6fac43fb32aa104(127.8.97.129:59546)]}] 01:30:25.574 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.97.190:34778, [2ms] Sub RPC GetTableLocations: received response from server master-127.8.97.190:34778: OK, deferred=null) after lookup 01:30:25.575 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=retry RPC@234388434 returned Deferred@94920192(state=PENDING, result=null, callback=(continuation of Deferred@721075579 after retry RPC@234388434), errback=(continuation of Deferred@721075579 after retry RPC@234388434)), so the following Deferred is getting paused: Deferred@721075579(state=PAUSED, result=Deferred@94920192, callback=org.apache.kudu.client.AsyncKuduSession$2@5ef9821e -> passthrough -> wakeup thread pool-489-thread-1, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@109154f8 -> wakeup thread pool-489-thread-1) 01:30:25.576 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:2118) Learned about tablet 80798175d30f473995c98f40d358813b for table 'testMultipleSessions' with partition [, ) 01:30:25.576 [DEBUG - New I/O worker #2172] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=80798175d30f473995c98f40d358813b@[49c72263ff5c41848addbf216e68983f(127.8.97.130:47565),6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775)[L],b18778d645364e8bb6fac43fb32aa104(127.8.97.129:59546)]}] 01:30:25.576 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.97.190:34778, [5ms] Sub RPC GetTableLocations: received response from server master-127.8.97.190:34778: OK, deferred=null) after lookup 01:30:25.576 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=retry RPC@1468940468 returned Deferred@677391963(state=PENDING, result=null, callback=(continuation of Deferred@983281723 after retry RPC@1468940468), errback=(continuation of Deferred@983281723 after retry RPC@1468940468)), so the following Deferred is getting paused: Deferred@983281723(state=PAUSED, result=Deferred@677391963, callback=org.apache.kudu.client.AsyncKuduSession$2@20b71504 -> passthrough -> wakeup thread pool-489-thread-2, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5997a46 -> wakeup thread pool-489-thread-2) 01:30:25.579 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:2118) Learned about tablet 80798175d30f473995c98f40d358813b for table 'testMultipleSessions' with partition [, ) 01:30:25.579 [DEBUG - New I/O worker #2172] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=80798175d30f473995c98f40d358813b@[49c72263ff5c41848addbf216e68983f(127.8.97.130:47565),6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775)[L],b18778d645364e8bb6fac43fb32aa104(127.8.97.129:59546)]}] 01:30:25.579 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=6), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.97.190:34778, [6ms] Sub RPC GetTableLocations: received response from server master-127.8.97.190:34778: OK, deferred=null) after lookup 01:30:25.580 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=retry RPC@748850776 returned Deferred@1264576927(state=PENDING, result=null, callback=(continuation of Deferred@1406716613 after retry RPC@748850776), errback=(continuation of Deferred@1406716613 after retry RPC@748850776)), so the following Deferred is getting paused: Deferred@1406716613(state=PAUSED, result=Deferred@1264576927, callback=org.apache.kudu.client.AsyncKuduSession$2@1dddd1b2 -> passthrough -> wakeup thread pool-489-thread-3, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@25b55c1f -> wakeup thread pool-489-thread-3) 01:30:25.580 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:2118) Learned about tablet 80798175d30f473995c98f40d358813b for table 'testMultipleSessions' with partition [, ) 01:30:25.580 [DEBUG - New I/O worker #2172] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=80798175d30f473995c98f40d358813b@[49c72263ff5c41848addbf216e68983f(127.8.97.130:47565),6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775)[L],b18778d645364e8bb6fac43fb32aa104(127.8.97.129:59546)]}] 01:30:25.580 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=7), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.97.190:34778, [7ms] Sub RPC GetTableLocations: received response from server master-127.8.97.190:34778: OK, deferred=null) after lookup 01:30:25.580 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=retry RPC@286415399 returned Deferred@76096546(state=PENDING, result=null, callback=(continuation of Deferred@2007639360 after retry RPC@286415399), errback=(continuation of Deferred@2007639360 after retry RPC@286415399)), so the following Deferred is getting paused: Deferred@2007639360(state=PAUSED, result=Deferred@76096546, callback=org.apache.kudu.client.AsyncKuduSession$2@7870ca0e -> passthrough -> wakeup thread pool-489-thread-4, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7bb06445 -> wakeup thread pool-489-thread-4) 01:30:25.581 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:2118) Learned about tablet 80798175d30f473995c98f40d358813b for table 'testMultipleSessions' with partition [, ) 01:30:25.581 [DEBUG - New I/O worker #2172] (TableLocationsCache.java:188) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=80798175d30f473995c98f40d358813b@[49c72263ff5c41848addbf216e68983f(127.8.97.130:47565),6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775)[L],b18778d645364e8bb6fac43fb32aa104(127.8.97.129:59546)]}] 01:30:25.583 [DEBUG - New I/O worker #2172] (AsyncKuduClient.java:1277) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, DeadlineTracker(timeout=30000, elapsed=7), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.97.190:34778, [8ms] Sub RPC GetTableLocations: received response from server master-127.8.97.190:34778: OK, deferred=null) after lookup 01:30:25.583 [DEBUG - New I/O worker #2172] (Deferred.java:1330) callback=retry RPC@919992101 returned Deferred@284378266(state=PENDING, result=null, callback=(continuation of Deferred@1848499248 after retry RPC@919992101), errback=(continuation of Deferred@1848499248 after retry RPC@919992101)), so the following Deferred is getting paused: Deferred@1848499248(state=PAUSED, result=Deferred@284378266, callback=org.apache.kudu.client.AsyncKuduSession$2@16471ffc -> passthrough -> wakeup thread pool-489-thread-5, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f32aece -> wakeup thread pool-489-thread-5) 01:30:25.587 [DEBUG - New I/O worker #2173] (Negotiator.java:764) Authenticated connection [id: 0xb536b0d8, /127.8.97.131:59331 => /127.8.97.131:52775] using TOKEN/null 01:30:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.652700 9023 raft_consensus.cc:1181] T 80798175d30f473995c98f40d358813b P 49c72263ff5c41848addbf216e68983f [term 1 FOLLOWER]: Refusing update from remote peer 6928ede89be54bbab71eedab94e61d67: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:30:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.654768 8892 raft_consensus.cc:1181] T 80798175d30f473995c98f40d358813b P b18778d645364e8bb6fac43fb32aa104 [term 1 FOLLOWER]: Refusing update from remote peer 6928ede89be54bbab71eedab94e61d67: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch) 01:30:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.657577 9245 consensus_queue.cc:984] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [LEADER]: Connected to new peer: Peer: permanent_uuid: "49c72263ff5c41848addbf216e68983f" member_type: VOTER last_known_addr { host: "127.8.97.130" port: 47565 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s 01:30:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.658499 9247 consensus_queue.cc:984] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b18778d645364e8bb6fac43fb32aa104" member_type: VOTER last_known_addr { host: "127.8.97.129" port: 59546 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s 01:30:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:25.713316 9312 mvcc.cc:203] Tried to move safe_time back from 6351129090643689472 to 6351129089728573440. Current Snapshot: MvccSnapshot[committed={T|T < 6351129090643689472}] 01:30:26.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:26.424355 9306 mvcc.cc:203] Tried to move safe_time back from 6351129093791670272 to 6351129089728573440. Current Snapshot: MvccSnapshot[committed={T|T < 6351129093791670272 or (T in {6351129093791670272})}] 01:30:27.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:27.970206 8667 debug-util.cc:397] Leaking SignalData structure 0x7b0800029080 after lost signal to thread 8655 01:30:27.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:27.970966 8667 debug-util.cc:397] Leaking SignalData structure 0x7b0800078400 after lost signal to thread 8713 01:30:35.027 [DEBUG - New I/O worker #2170] (Connection.java:429) [peer master-127.8.97.189:35516(127.8.97.189:35516)] encountered a read timeout; closing the channel 01:30:35.027 [DEBUG - New I/O worker #2170] (Connection.java:688) [peer master-127.8.97.189:35516(127.8.97.189:35516)] cleaning up while in state READY due to: [peer master-127.8.97.189:35516(127.8.97.189:35516)] encountered a read timeout; closing the channel 01:30:35.027 [DEBUG - New I/O worker #2171] (Connection.java:429) [peer master-127.8.97.188:45488(127.8.97.188:45488)] encountered a read timeout; closing the channel 01:30:35.028 [DEBUG - New I/O worker #2171] (Connection.java:688) [peer master-127.8.97.188:45488(127.8.97.188:45488)] cleaning up while in state READY due to: [peer master-127.8.97.188:45488(127.8.97.188:45488)] encountered a read timeout; closing the channel 01:30:35.588 [DEBUG - New I/O worker #2172] (Connection.java:429) [peer master-127.8.97.190:34778(127.8.97.190:34778)] encountered a read timeout; closing the channel 01:30:35.588 [DEBUG - New I/O worker #2172] (Connection.java:688) [peer master-127.8.97.190:34778(127.8.97.190:34778)] cleaning up while in state READY due to: [peer master-127.8.97.190:34778(127.8.97.190:34778)] encountered a read timeout; closing the channel 01:30:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.355496 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1242, attempt_no=1}) took 1028ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":1165,"num_ops":1,"prepare.queue_time_us":848096,"prepare.run_cpu_time_us":12907,"prepare.run_wall_time_us":100197,"replication_time_us":74163,"row_lock_wait_count":1,"row_lock_wait_us":74182,"spinlock_wait_cycles":1133312}]]} 01:30:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.401886 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1243, attempt_no=1}) took 1067ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":432,"num_ops":1,"prepare.queue_time_us":940597,"prepare.run_cpu_time_us":8309,"prepare.run_wall_time_us":81207,"replication_time_us":42148,"row_lock_wait_count":1,"row_lock_wait_us":63913,"spinlock_wait_cycles":251648}]]} 01:30:40.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.421337 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1244, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":247,"num_ops":1,"prepare.queue_time_us":1008437,"prepare.run_cpu_time_us":10288,"prepare.run_wall_time_us":46252,"replication_time_us":15893,"row_lock_wait_count":1,"row_lock_wait_us":31912,"spinlock_wait_cycles":1657216}]]} 01:30:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.437187 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1245, attempt_no=1}) took 1077ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":228,"num_ops":1,"prepare.queue_time_us":1043094,"prepare.run_cpu_time_us":2400,"prepare.run_wall_time_us":18128,"replication_time_us":13508,"row_lock_wait_count":1,"row_lock_wait_us":15792,"spinlock_wait_cycles":113024}]]} 01:30:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.456797 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1246, attempt_no=1}) took 1089ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":532,"num_ops":1,"prepare.queue_time_us":1053525,"prepare.run_cpu_time_us":1956,"prepare.run_wall_time_us":15591,"replication_time_us":12251,"row_lock_wait_count":1,"row_lock_wait_us":13610,"spinlock_wait_cycles":113024}]]} 01:30:40.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.463096 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1247, attempt_no=1}) took 1082ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":106,"num_ops":1,"prepare.queue_time_us":1056830,"prepare.run_cpu_time_us":1612,"prepare.run_wall_time_us":15076,"replication_time_us":9394,"row_lock_wait_count":1,"row_lock_wait_us":13485}]]} 01:30:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.474453 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1248, attempt_no=1}) took 1085ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":106,"mutex_wait_us":54,"num_ops":1,"prepare.queue_time_us":1062642,"prepare.run_cpu_time_us":1512,"prepare.run_wall_time_us":10772,"replication_time_us":9793,"row_lock_wait_count":1,"row_lock_wait_us":9278,"spinlock_wait_cycles":346496}]]} 01:30:40.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.486623 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1249, attempt_no=1}) took 1083ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":86,"num_ops":1,"prepare.queue_time_us":1059944,"prepare.run_cpu_time_us":1440,"prepare.run_wall_time_us":10907,"replication_time_us":9364,"row_lock_wait_count":1,"row_lock_wait_us":9484,"spinlock_wait_cycles":93568}]]} 01:30:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.496122 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1250, attempt_no=1}) took 1084ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":115,"num_ops":1,"prepare.queue_time_us":1061801,"prepare.run_cpu_time_us":1375,"prepare.run_wall_time_us":10511,"replication_time_us":9752,"row_lock_wait_count":1,"row_lock_wait_us":9155,"spinlock_wait_cycles":181504}]]} 01:30:40.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.505163 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1251, attempt_no=1}) took 1082ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":348,"num_ops":1,"prepare.queue_time_us":1062187,"prepare.run_cpu_time_us":1452,"prepare.run_wall_time_us":11178,"replication_time_us":7321,"row_lock_wait_count":1,"row_lock_wait_us":9741,"spinlock_wait_cycles":39808}]]} 01:30:40.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.517087 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1252, attempt_no=1}) took 1084ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":158,"num_ops":1,"prepare.queue_time_us":1063481,"prepare.run_cpu_time_us":1404,"prepare.run_wall_time_us":9527,"replication_time_us":8059,"row_lock_wait_count":1,"row_lock_wait_us":7961}]]} 01:30:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.524363 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1253, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":100,"num_ops":1,"prepare.queue_time_us":1054331,"prepare.run_cpu_time_us":1345,"prepare.run_wall_time_us":9091,"replication_time_us":7162,"row_lock_wait_count":1,"row_lock_wait_us":7583}]]} 01:30:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.536648 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1254, attempt_no=1}) took 1077ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":184,"num_ops":1,"prepare.queue_time_us":1055936,"prepare.run_cpu_time_us":1348,"prepare.run_wall_time_us":8988,"replication_time_us":8904,"row_lock_wait_count":1,"row_lock_wait_us":7669,"spinlock_wait_cycles":102144}]]} 01:30:40.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.545202 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1255, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":119,"num_ops":1,"prepare.queue_time_us":1052335,"prepare.run_cpu_time_us":1272,"prepare.run_wall_time_us":10164,"replication_time_us":8851,"row_lock_wait_count":1,"row_lock_wait_us":8908}]]} 01:30:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.555866 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1256, attempt_no=1}) took 1070ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":115,"num_ops":1,"prepare.queue_time_us":1049081,"prepare.run_cpu_time_us":1485,"prepare.run_wall_time_us":10627,"replication_time_us":7518,"row_lock_wait_count":1,"row_lock_wait_us":9173,"spinlock_wait_cycles":81152}]]} 01:30:40.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.567175 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1257, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":142,"num_ops":1,"prepare.queue_time_us":1051281,"prepare.run_cpu_time_us":1488,"prepare.run_wall_time_us":10527,"replication_time_us":7667,"row_lock_wait_count":1,"row_lock_wait_us":9091,"spinlock_wait_cycles":61952}]]} 01:30:40.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.576696 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1258, attempt_no=1}) took 1072ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":861,"num_ops":1,"prepare.queue_time_us":1050864,"prepare.run_cpu_time_us":1407,"prepare.run_wall_time_us":9283,"replication_time_us":7818,"row_lock_wait_count":1,"row_lock_wait_us":7891,"spinlock_wait_cycles":1120512}]]} 01:30:40.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.585248 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1259, attempt_no=1}) took 1069ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":754,"num_ops":1,"prepare.queue_time_us":1049661,"prepare.run_cpu_time_us":1343,"prepare.run_wall_time_us":9719,"replication_time_us":7760,"row_lock_wait_count":1,"row_lock_wait_us":8394}]]} 01:30:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.596066 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1260, attempt_no=1}) took 1067ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":90,"num_ops":1,"prepare.queue_time_us":1045965,"prepare.run_cpu_time_us":1453,"prepare.run_wall_time_us":10058,"replication_time_us":7713,"row_lock_wait_count":1,"row_lock_wait_us":8639,"spinlock_wait_cycles":67456}]]} 01:30:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.609859 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1261, attempt_no=1}) took 1070ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":180,"num_ops":1,"prepare.queue_time_us":1045617,"prepare.run_cpu_time_us":1564,"prepare.run_wall_time_us":9097,"replication_time_us":10732,"row_lock_wait_count":1,"row_lock_wait_us":7551,"spinlock_wait_cycles":196992}]]} 01:30:40.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.624603 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1262, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":174,"num_ops":1,"prepare.queue_time_us":1042818,"prepare.run_cpu_time_us":1485,"prepare.run_wall_time_us":12465,"replication_time_us":13241,"row_lock_wait_count":1,"row_lock_wait_us":10997}]]} 01:30:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.632239 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1263, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":178,"num_ops":1,"prepare.queue_time_us":1047808,"prepare.run_cpu_time_us":1479,"prepare.run_wall_time_us":15146,"replication_time_us":6621,"row_lock_wait_count":1,"row_lock_wait_us":13685}]]} 01:30:40.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.645242 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1264, attempt_no=1}) took 1075ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":90,"num_ops":1,"prepare.queue_time_us":1052221,"prepare.run_cpu_time_us":1402,"prepare.run_wall_time_us":8264,"replication_time_us":10150,"row_lock_wait_count":1,"row_lock_wait_us":6876}]]} 01:30:40.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.651959 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1265, attempt_no=1}) took 1069ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":119,"num_ops":1,"prepare.queue_time_us":1048055,"prepare.run_cpu_time_us":1518,"prepare.run_wall_time_us":11727,"replication_time_us":8083,"row_lock_wait_count":1,"row_lock_wait_us":10228}]]} 01:30:40.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.662988 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1266, attempt_no=1}) took 1067ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":119,"num_ops":1,"prepare.queue_time_us":1046457,"prepare.run_cpu_time_us":1512,"prepare.run_wall_time_us":9700,"replication_time_us":9559,"row_lock_wait_count":1,"row_lock_wait_us":8205,"spinlock_wait_cycles":548608}]]} 01:30:40.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.674307 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1267, attempt_no=1}) took 1066ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":220,"num_ops":1,"prepare.queue_time_us":1043899,"prepare.run_cpu_time_us":1328,"prepare.run_wall_time_us":10915,"replication_time_us":9152,"row_lock_wait_count":1,"row_lock_wait_us":9601}]]} 01:30:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.684437 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1268, attempt_no=1}) took 1063ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":197,"mutex_wait_us":37,"num_ops":1,"prepare.queue_time_us":1041896,"prepare.run_cpu_time_us":1300,"prepare.run_wall_time_us":10549,"replication_time_us":7954,"row_lock_wait_count":1,"row_lock_wait_us":9265,"spinlock_wait_cycles":943872}]]} 01:30:40.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.692832 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1269, attempt_no=1}) took 1061ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":117,"num_ops":1,"prepare.queue_time_us":1042486,"prepare.run_cpu_time_us":1650,"prepare.run_wall_time_us":10225,"raft.queue_time_us":593,"raft.run_cpu_time_us":10,"raft.run_wall_time_us":10,"replication_time_us":7900,"row_lock_wait_count":1,"row_lock_wait_us":7949,"spinlock_wait_cycles":1834368}]]} 01:30:40.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.704663 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1270, attempt_no=1}) took 1058ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":111,"num_ops":1,"prepare.queue_time_us":1037431,"prepare.run_cpu_time_us":1504,"prepare.run_wall_time_us":8464,"replication_time_us":10505,"row_lock_wait_count":1,"row_lock_wait_us":7014,"spinlock_wait_cycles":55168}]]} 01:30:40.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.716871 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1271, attempt_no=1}) took 1059ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":122,"num_ops":1,"prepare.queue_time_us":1035261,"prepare.run_cpu_time_us":1658,"prepare.run_wall_time_us":12943,"replication_time_us":10430,"row_lock_wait_count":1,"row_lock_wait_us":9479,"spinlock_wait_cycles":17152}]]} 01:30:40.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.729941 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1272, attempt_no=1}) took 1062ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":310,"num_ops":1,"prepare.queue_time_us":1038115,"prepare.run_cpu_time_us":1562,"prepare.run_wall_time_us":11940,"replication_time_us":10967,"row_lock_wait_count":1,"row_lock_wait_us":9688}]]} 01:30:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.744663 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1273, attempt_no=1}) took 1065ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":232,"num_ops":1,"prepare.queue_time_us":1038596,"prepare.run_cpu_time_us":1918,"prepare.run_wall_time_us":16688,"replication_time_us":10335,"row_lock_wait_count":1,"row_lock_wait_us":11842,"spinlock_wait_cycles":3367168}]]} 01:30:40.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.757097 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1274, attempt_no=1}) took 1065ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":693,"num_ops":1,"prepare.queue_time_us":1042905,"prepare.run_cpu_time_us":2378,"prepare.run_wall_time_us":15668,"replication_time_us":9303,"row_lock_wait_count":1,"row_lock_wait_us":8464,"spinlock_wait_cycles":8187904}]]} 01:30:40.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.772264 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1275, attempt_no=1}) took 1072ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":228,"num_ops":1,"prepare.queue_time_us":1050162,"prepare.run_cpu_time_us":1567,"prepare.run_wall_time_us":8348,"replication_time_us":13150,"row_lock_wait_count":1,"row_lock_wait_us":5325,"spinlock_wait_cycles":24960}]]} 01:30:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.784605 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1276, attempt_no=1}) took 1073ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":301,"num_ops":1,"prepare.queue_time_us":1047795,"prepare.run_cpu_time_us":2208,"prepare.run_wall_time_us":18439,"replication_time_us":7384,"row_lock_wait_count":1,"row_lock_wait_us":12966,"spinlock_wait_cycles":3586816}]]} 01:30:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.799597 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1277, attempt_no=1}) took 1077ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":253,"num_ops":1,"prepare.queue_time_us":1055534,"prepare.run_cpu_time_us":1857,"prepare.run_wall_time_us":10683,"raft.queue_time_us":78,"raft.run_cpu_time_us":601,"raft.run_wall_time_us":601,"replication_time_us":7377,"row_lock_wait_count":1,"row_lock_wait_us":6418,"spinlock_wait_cycles":1528064}]]} 01:30:40.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.813064 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1278, attempt_no=1}) took 1079ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":211,"num_ops":1,"prepare.queue_time_us":1054626,"prepare.run_cpu_time_us":2031,"prepare.run_wall_time_us":13988,"raft.queue_time_us":84,"raft.run_cpu_time_us":720,"raft.run_wall_time_us":741,"replication_time_us":7388,"row_lock_wait_count":1,"row_lock_wait_us":10089,"spinlock_wait_cycles":2767488}]]} 01:30:40.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.832866 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1279, attempt_no=1}) took 1088ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":257,"num_ops":1,"prepare.queue_time_us":1058271,"prepare.run_cpu_time_us":1742,"prepare.run_wall_time_us":13522,"raft.queue_time_us":91,"raft.run_cpu_time_us":715,"raft.run_wall_time_us":737,"replication_time_us":16547,"row_lock_wait_count":1,"row_lock_wait_us":7207,"spinlock_wait_cycles":302336}]]} 01:30:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.854183 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1280, attempt_no=1}) took 1093ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":191,"num_ops":1,"prepare.queue_time_us":1055800,"prepare.run_cpu_time_us":2020,"prepare.run_wall_time_us":20448,"replication_time_us":16543,"row_lock_wait_count":1,"row_lock_wait_us":14647,"spinlock_wait_cycles":7411584}]]} 01:30:40.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.865098 9373 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1281, attempt_no=1}) took 1090ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":1635,"num_ops":1,"prepare.queue_time_us":1062488,"prepare.run_cpu_time_us":1667,"prepare.run_wall_time_us":15116,"replication_time_us":10093,"row_lock_wait_count":1,"row_lock_wait_us":12680}]]} 01:30:40.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.866796 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1282, attempt_no=1}) took 1088ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":1550,"num_ops":1,"prepare.queue_time_us":1073809,"prepare.run_cpu_time_us":1511,"prepare.run_wall_time_us":2241,"replication_time_us":8002,"spinlock_wait_cycles":155904,"thread_start_us":959,"threads_started":1}]]} 01:30:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.886034 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1283, attempt_no=1}) took 1094ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":686,"num_ops":1,"prepare.queue_time_us":1066659,"prepare.run_cpu_time_us":1510,"prepare.run_wall_time_us":13407,"replication_time_us":10727,"row_lock_wait_count":1,"row_lock_wait_us":11008,"spinlock_wait_cycles":77056}]]} 01:30:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.892179 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1284, attempt_no=1}) took 1094ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":84,"num_ops":1,"prepare.queue_time_us":1070171,"prepare.run_cpu_time_us":1681,"prepare.run_wall_time_us":14675,"replication_time_us":9014,"row_lock_wait_count":1,"row_lock_wait_us":12116}]]} 01:30:40.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.902007 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1285, attempt_no=1}) took 1085ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":493,"num_ops":1,"prepare.queue_time_us":1065935,"prepare.run_cpu_time_us":1552,"prepare.run_wall_time_us":10483,"replication_time_us":7721,"row_lock_wait_count":1,"row_lock_wait_us":6758}]]} 01:30:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.918323 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1286, attempt_no=1}) took 1084ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":139,"num_ops":1,"prepare.queue_time_us":1059267,"prepare.run_cpu_time_us":1634,"prepare.run_wall_time_us":9763,"replication_time_us":13943,"row_lock_wait_count":1,"row_lock_wait_us":7329}]]} 01:30:40.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.934398 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1287, attempt_no=1}) took 1092ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":4711,"num_ops":1,"prepare.queue_time_us":1060498,"prepare.run_cpu_time_us":1763,"prepare.run_wall_time_us":15045,"replication_time_us":9662,"row_lock_wait_count":1,"row_lock_wait_us":11772,"spinlock_wait_cycles":43392}]]} 01:30:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.946563 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1288, attempt_no=1}) took 1092ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":323,"num_ops":1,"prepare.queue_time_us":1063989,"prepare.run_cpu_time_us":1581,"prepare.run_wall_time_us":15922,"raft.queue_time_us":621,"raft.run_cpu_time_us":579,"raft.run_wall_time_us":579,"replication_time_us":8374,"row_lock_wait_count":1,"row_lock_wait_us":14374,"spinlock_wait_cycles":196096}]]} 01:30:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.957242 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1289, attempt_no=1}) took 1095ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":246,"num_ops":1,"prepare.queue_time_us":1072799,"prepare.run_cpu_time_us":2315,"prepare.run_wall_time_us":14265,"replication_time_us":8893,"row_lock_wait_count":1,"row_lock_wait_us":8289,"spinlock_wait_cycles":9535232}]]} 01:30:40.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.967880 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1290, attempt_no=1}) took 1096ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":226,"num_ops":1,"prepare.queue_time_us":1077285,"prepare.run_cpu_time_us":1303,"prepare.run_wall_time_us":8175,"replication_time_us":9146,"row_lock_wait_count":1,"row_lock_wait_us":6892}]]} 01:30:40.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.980792 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1291, attempt_no=1}) took 1097ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":283,"num_ops":1,"prepare.queue_time_us":1073158,"prepare.run_cpu_time_us":1594,"prepare.run_wall_time_us":10705,"replication_time_us":10599,"row_lock_wait_count":1,"row_lock_wait_us":9146,"spinlock_wait_cycles":171904}]]} 01:30:40.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:40.992285 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1292, attempt_no=1}) took 1096ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":208,"num_ops":1,"prepare.queue_time_us":1071404,"prepare.run_cpu_time_us":1766,"prepare.run_wall_time_us":13055,"raft.queue_time_us":722,"raft.run_cpu_time_us":12,"raft.run_wall_time_us":12,"replication_time_us":9360,"row_lock_wait_count":1,"row_lock_wait_us":11221,"spinlock_wait_cycles":624896}]]} 01:30:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.005342 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1293, attempt_no=1}) took 1092ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":232,"num_ops":1,"prepare.queue_time_us":1068365,"prepare.run_cpu_time_us":1620,"prepare.run_wall_time_us":11111,"replication_time_us":9190,"row_lock_wait_count":1,"row_lock_wait_us":9543,"spinlock_wait_cycles":37248}]]} 01:30:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.017789 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1294, attempt_no=1}) took 1097ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":217,"num_ops":1,"prepare.queue_time_us":1071792,"prepare.run_cpu_time_us":1450,"prepare.run_wall_time_us":10976,"replication_time_us":9500,"row_lock_wait_count":1,"row_lock_wait_us":9555}]]} 01:30:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.027808 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1295, attempt_no=1}) took 1098ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":221,"num_ops":1,"prepare.queue_time_us":1073093,"prepare.run_cpu_time_us":1777,"prepare.run_wall_time_us":14668,"replication_time_us":8090,"row_lock_wait_count":1,"row_lock_wait_us":12779,"spinlock_wait_cycles":678912}]]} 01:30:41.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.041066 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1296, attempt_no=1}) took 1093ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":182,"num_ops":1,"prepare.queue_time_us":1070365,"prepare.run_cpu_time_us":1565,"prepare.run_wall_time_us":9609,"replication_time_us":8809,"row_lock_wait_count":1,"row_lock_wait_us":8073,"spinlock_wait_cycles":325504}]]} 01:30:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.049010 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1297, attempt_no=1}) took 1089ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":186,"num_ops":1,"prepare.queue_time_us":1067595,"prepare.run_cpu_time_us":1528,"prepare.run_wall_time_us":11774,"replication_time_us":7236,"row_lock_wait_count":1,"row_lock_wait_us":10265,"spinlock_wait_cycles":27904}]]} 01:30:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.060480 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1298, attempt_no=1}) took 1083ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":193,"num_ops":1,"prepare.queue_time_us":1061949,"prepare.run_cpu_time_us":1561,"prepare.run_wall_time_us":9134,"replication_time_us":8623,"row_lock_wait_count":1,"row_lock_wait_us":7632,"spinlock_wait_cycles":107776}]]} 01:30:41.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.069730 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1299, attempt_no=1}) took 1070ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":109,"num_ops":1,"prepare.queue_time_us":1047863,"prepare.run_cpu_time_us":1337,"prepare.run_wall_time_us":9747,"replication_time_us":9316,"row_lock_wait_count":1,"row_lock_wait_us":8428,"spinlock_wait_cycles":1843584}]]} 01:30:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:30:41.081388 9364 rpcz_store.cc:269] Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=1300, attempt_no=1}) took 1064ms. Request Metrics: {"child_traces":[["txn",{"apply.queue_time_us":546,"num_ops":1,"prepare.queue_time_us":1016832,"prepare.run_cpu_time_us":1494,"prepare.run_wall_time_us":10997,"replication_time_us":8265,"row_lock_wait_count":1,"row_lock_wait_us":9520}]]} 01:30:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:44.053825 8732 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 8720 01:30:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:30:44.055198 8732 debug-util.cc:397] Leaking SignalData structure 0x7b080008c840 after lost signal to thread 8778 01:31:09.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:09.865092 9089 debug-util.cc:397] Leaking SignalData structure 0x7b0800029060 after lost signal to thread 9077 01:31:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:09.866178 9089 debug-util.cc:397] Leaking SignalData structure 0x7b08000ca000 after lost signal to thread 9195 01:31:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:15.531375 8603 debug-util.cc:397] Leaking SignalData structure 0x7b0800029080 after lost signal to thread 8591 01:31:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:15.532846 8603 debug-util.cc:397] Leaking SignalData structure 0x7b080000c660 after lost signal to thread 8649 01:31:25.009 [DEBUG - Time-limited test] (Deferred.java:1189) While joining Deferred@1966094667(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$2@6bba1e1d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2810943a -> wakeup thread Time-limited test): interrupted 01:31:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.016257 9086 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=5160, attempt_no=1}) could send its response 01:31:25.017 [DEBUG - Test worker] (AsyncKuduClient.java:2285) Releasing all remaining resources 01:31:25.017 [DEBUG - New I/O worker #2173] (Connection.java:688) [peer 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775)] cleaning up while in state READY due to: connection disconnected 01:31:25.017 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.017 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.019 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.019 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.019 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.020 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.020 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.020 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.021 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.021 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.021 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.022 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.022 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.022 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.022 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.023 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.024 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.029 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.030 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.030 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.035 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.035 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:31:25.022964 8582 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 8781 01:31:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.034469 9086 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=5161, attempt_no=1}) could send its response 01:31:25.035 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.040 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.040 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.040 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.048332 9086 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=5162, attempt_no=1}) could send its response 01:31:25.049 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.049 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.050 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.056 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.056 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.057 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.065 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.065 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.064000 9086 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=5163, attempt_no=1}) could send its response 01:31:25.066 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.072 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.080 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.081 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.084 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.073897 8666 connection.cc:647] server connection from 127.8.97.129:54761 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.129:54761): Connection reset by peer (error 104) 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.077162 8601 connection.cc:647] server connection from 127.8.97.129:37420 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.129:37420): Connection reset by peer (error 104) 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.077211 8730 connection.cc:647] server connection from 127.8.97.129:35179 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.129:35179): Connection reset by peer (error 104) 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:31:25.079555 8582 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 8939 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.080677 9086 connection.cc:554] Connection torn down before Call kudu.tserver.TabletServerService.Write from 127.8.97.131:59331 (ReqId={client: 566795d7ae214f2c9002482035fb54b4, seq_no=5164, attempt_no=1}) could send its response 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.081406 9086 connection.cc:647] client connection to 127.8.97.129:59546 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.081777 9086 consensus_peers.cc:457] T 80798175d30f473995c98f40d358813b P 6928ede89be54bbab71eedab94e61d67 -> Peer b18778d645364e8bb6fac43fb32aa104 (127.8.97.129:59546): Couldn't send request to peer b18778d645364e8bb6fac43fb32aa104. Status: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104). This is attempt 1: this message will repeat every 5th retry. 01:31:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.082653 9086 connection.cc:320] Error closing socket: Network error: TlsSocket::Close: Broken pipe 01:31:25.084 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.085 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.086 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.086 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.086 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.086 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.086 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.087 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.087 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.087 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.087 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.088 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.088 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.088 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.088 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.089 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.089 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.089 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.089 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.089 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.090 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.090 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.090 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.091 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.091 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.091 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.091 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.091 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.092 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.092 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.092 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.092 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.093 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.093 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.093 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.094 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.094 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.094 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.094 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.095 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.095 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.095 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.095 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.096 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.096 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.096 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.097 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.097 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.097 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.097 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.098 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.098 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.098 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.099 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.099 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.099 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.100 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.100 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.100 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.101 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.101 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.101 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.101 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.102 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.102 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.102 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.102 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.103 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.103 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.103 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.104 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.104 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.104 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.104 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.105 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.105 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.106 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.106 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.106 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.107 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.107 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.107 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.108 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.108 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.108 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.108 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.109 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.109 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.109 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.110 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.110 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.110 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.111 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.111 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.111 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.112 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.112 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.113 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.113 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.113 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.113 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.114 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.114 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.114 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.114 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.114 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.114 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.115 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.115 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.115 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.115 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.115 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.115 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.116 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.116 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.116 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.116 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.116 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.116 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.117 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.117 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.117 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.117 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.118 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.118 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.118 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.118 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.118 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.118 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.118 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.119 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.119 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.119 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.119 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.120 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.120 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.121 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.121 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.121 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.122 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.122 [INFO - New I/O worker #2173] (AsyncKuduClient.java:1970) Invalidating location 6928ede89be54bbab71eedab94e61d67(127.8.97.131:52775) for tablet 80798175d30f473995c98f40d358813b: connection disconnected 01:31:25.122 [DEBUG - New I/O worker #2173] (RemoteTablet.java:131) tablet 80798175d30f473995c98f40d358813b already removed ts 6928ede89be54bbab71eedab94e61d67, size left is 2 01:31:25.122 [WARN - New I/O worker #2173] (AsyncKuduClient.java:2355) Failed to schedule timer. Ignore this if we're shutting down. java.lang.IllegalStateException: cannot be started once stopped at org.jboss.netty.util.HashedWheelTimer.start(HashedWheelTimer.java:279) at org.jboss.netty.util.HashedWheelTimer.newTimeout(HashedWheelTimer.java:337) at org.apache.kudu.client.AsyncKuduClient.newTimeout(AsyncKuduClient.java:2350) at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:1959) at org.apache.kudu.client.AsyncKuduClient.handleRetryableError(AsyncKuduClient.java:1894) at org.apache.kudu.client.AsyncKuduClient.handleTabletNotFound(AsyncKuduClient.java:1878) at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:389) at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:216) at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:59) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:131) at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:127) at org.apache.kudu.client.Connection.cleanup(Connection.java:707) at org.apache.kudu.client.Connection.channelDisconnected(Connection.java:251) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.apache.kudu.client.Connection.handleUpstream(Connection.java:243) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.oneone.OneToOneDecoder.handleUpstream(OneToOneDecoder.java:60) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493) at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.Channels$4.run(Channels.java:386) at org.jboss.netty.channel.socket.ChannelRunnableWrapper.run(ChannelRunnableWrapper.java:40) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:391) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:315) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 01:31:25.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.145063 8730 connection.cc:647] server connection from 127.8.97.130:51623 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.130:51623): Connection reset by peer (error 104) 01:31:25.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.145067 8664 connection.cc:647] server connection from 127.8.97.130:45190 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.130:45190): Connection reset by peer (error 104) 01:31:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.146636 8598 connection.cc:647] server connection from 127.8.97.130:37814 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.130:37814): Connection reset by peer (error 104) 01:31:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:31:25.147624 8582 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-tserver with pid 9069 01:31:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.147626 9086 connection.cc:647] client connection to 127.8.97.130:47565 recv error: Network error: failed to read from TLS socket (remote: 0.0.0.0:0): Connection reset by peer (error 104) 01:31:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.188102 8729 connection.cc:647] server connection from 127.8.97.131:57459 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.131:57459): Connection reset by peer (error 104) 01:31:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.188886 8599 connection.cc:647] server connection from 127.8.97.131:43660 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.131:43660): Connection reset by peer (error 104) 01:31:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) W0219 01:31:25.188942 8664 connection.cc:647] server connection from 127.8.97.131:60471 recv error: Network error: failed to read from TLS socket (remote: 127.8.97.131:60471): Connection reset by peer (error 104) 01:31:25.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:31:25.190232 8582 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 8590 01:31:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:31:25.212674 8582 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 8652 01:31:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:543) I0219 01:31:25.233605 8582 external_mini_cluster.cc:1131] Killing /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/bin/kudu-master with pid 8716 01:31:26.293 [ERROR - Test worker] (RetryRule.java:80) testMultipleSessions(org.apache.kudu.client.ITClientStress): failed attempt 4 org.junit.runners.model.TestTimedOutException: test timed out after 60000 milliseconds at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.stumbleupon.async.Deferred.doJoin(Deferred.java:1136) at com.stumbleupon.async.Deferred.join(Deferred.java:1019) at org.apache.kudu.client.KuduSession.apply(KuduSession.java:81) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:163) at org.apache.kudu.client.ITClientStress$3$1.call(ITClientStress.java:150) at org.apache.kudu.client.ITClientStress$1.run(ITClientStress.java:80) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:2022) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at org.apache.kudu.client.ITClientStress.runTasks(ITClientStress.java:86) at org.apache.kudu.client.ITClientStress.testMultipleSessions(ITClientStress.java:147) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) 01:31:26.294 [ERROR - Test worker] (RetryRule.java:83) testMultipleSessions(org.apache.kudu.client.ITClientStress): giving up after 4 attempts ]]>