Critical Description
Currently the second major reason why client-test is flaky is that we have a tsan race on master.
Check "fastdebug tsan" failures here: http://a1228.halxg.cloudera.com:8080/test_drilldown?test_name=client-test
TSAN trace:
==================
WARNING: ThreadSanitizer: data race (pid=17562)
Write of size 8 at 0x7d040000fc00 by thread T85 (mutexes: write M345777):
#0 std::string::_M_data(char*) /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/basic_string.h:282 (client-test+0x0000000f7c0d)
#1 std::string::_M_mutate(unsigned long, unsigned long, unsigned long) /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/basic_string.tcc:489 (client-test+0x0000000f88d6)
#2 std::string::clear() /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/basic_string.h:702 (client-test+0x0000000f8ce8)
#3 kudu::master::SysTabletsEntryPB::Clear() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.cc:2419 (libmaster_proto.so+0x00000006503f)
#4 kudu::master::SysTabletsEntryPB::CopyFrom(kudu::master::SysTabletsEntryPB const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.cc:2712 (libmaster_proto.so+0x000000065b8b)
#5 kudu::master::SysTabletsEntryPB::operator=(kudu::master::SysTabletsEntryPB const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.h:498 (libmaster.so+0x0000000db850)
#6 kudu::master::PersistentTabletInfo::operator=(kudu::master::PersistentTabletInfo const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.h:47 (libmaster.so+0x0000000db820)
#7 void std::swap<kudu::master::PersistentTabletInfo>(kudu::master::PersistentTabletInfo&, kudu::master::PersistentTabletInfo&) /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/move.h:82 (libmaster.so+0x0000000db750)
#8 kudu::CowObject<kudu::master::PersistentTabletInfo>::CommitMutation() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/cow_object.h:56 (libmaster.so+0x0000000c25af)
#9 kudu::CowLock<kudu::master::PersistentTabletInfo>::Commit() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/cow_object.h:146 (libmaster.so+0x0000000c5839)
#10 kudu::master::CatalogManager::HandleReportedTablet(kudu::master::TSDescriptor*, kudu::master::ReportedTabletPB const&, kudu::master::ReportedTabletUpdatesPB*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:1144 (libmaster.so+0x0000000ba234)
#11 kudu::master::CatalogManager::ProcessTabletReport(kudu::master::TSDescriptor*, kudu::master::TabletReportPB const&, kudu::master::TabletReportUpdatesPB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:1011 (libmaster.so+0x0000000b91df)
#12 kudu::master::MasterServiceImpl::TSHeartbeat(kudu::master::TSHeartbeatRequestPB const*, kudu::master::TSHeartbeatResponsePB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master_service.cc:148 (libmaster.so+0x0000000fc4df)
#13 kudu::master::MasterServiceIf::Handle(kudu::rpc::InboundCall*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.service.cc:103 (libmaster_proto.so+0x0000000a1e2f)
#14 kudu::rpc::ServicePool::RunThread() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:156 (libkrpc.so+0x0000000c93ba)
#15 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000cc0ad)
#16 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246 (libkrpc.so+0x0000000cc01a)
#17 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkrpc.so+0x0000000cbfc3)
#18 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkrpc.so+0x0000000cbdc9)
#19 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012 (libtablet.so+0x000000211501)
#20 kudu::Thread::SuperviseThread(void*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:436 (libkudu_util.so+0x00000015566b)
Previous read of size 8 at 0x7d040000fc00 by thread T22:
#0 std::string::_M_data() const /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/basic_string.h:278 (client-test+0x0000000f79f9)
#1 std::string::_M_rep() const /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/basic_string.h:286 (client-test+0x0000000f7b29)
#2 std::string::assign(std::string const&) /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/basic_string.tcc:246 (client-test+0x000000105f4e)
#3 kudu::master::TabletLocationsPB::set_start_key(std::string const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.h:7117 (libmaster.so+0x0000000c7d6c)
#4 kudu::master::CatalogManager::BuildLocationsForTablet(scoped_refptr<kudu::master::TabletInfo> const&, kudu::master::TabletLocationsPB*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:1973 (libmaster.so+0x0000000bd7cf)
#5 kudu::master::CatalogManager::GetTableLocations(kudu::master::GetTableLocationsRequestPB const*, kudu::master::GetTableLocationsResponsePB*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:2057 (libmaster.so+0x0000000be039)
#6 kudu::master::MasterServiceImpl::GetTableLocations(kudu::master::GetTableLocationsRequestPB const*, kudu::master::GetTableLocationsResponsePB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master_service.cc:304 (libmaster.so+0x0000000fdcc6)
#7 kudu::master::MasterServiceIf::Handle(kudu::rpc::InboundCall*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.service.cc:220 (libmaster_proto.so+0x0000000a1ee4)
#8 kudu::rpc::ServicePool::RunThread() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:156 (libkrpc.so+0x0000000c93ba)
#9 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000cc0ad)
#10 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246 (libkrpc.so+0x0000000cc01a)
#11 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkrpc.so+0x0000000cbfc3)
#12 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkrpc.so+0x0000000cbdc9)
#13 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012 (libtablet.so+0x000000211501)
#14 kudu::Thread::SuperviseThread(void*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:436 (libkudu_util.so+0x00000015566b)
Location is heap block of size 8 at 0x7d040000fc00 allocated by thread T197:
#0 operator new(unsigned long) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:560 (client-test+0x00000005a32a)
#1 kudu::master::SysTabletsEntryPB::set_start_key(std::string const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.h:5606 (libmaster.so+0x0000000c32a3)
#2 kudu::master::SysTabletsEntryPB::MergeFrom(kudu::master::SysTabletsEntryPB const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.cc:2686 (libmaster_proto.so+0x000000064c28)
#3 kudu::master::SysTabletsEntryPB::CopyFrom(kudu::master::SysTabletsEntryPB const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.cc:2713 (libmaster_proto.so+0x000000065b96)
#4 kudu::master::SysTabletsEntryPB::operator=(kudu::master::SysTabletsEntryPB const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.pb.h:498 (libmaster.so+0x0000000db850)
#5 kudu::master::PersistentTabletInfo::operator=(kudu::master::PersistentTabletInfo const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.h:47 (libmaster.so+0x0000000db820)
#6 void std::swap<kudu::master::PersistentTabletInfo>(kudu::master::PersistentTabletInfo&, kudu::master::PersistentTabletInfo&) /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/move.h:82 (libmaster.so+0x0000000db750)
#7 kudu::CowObject<kudu::master::PersistentTabletInfo>::CommitMutation() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/cow_object.h:56 (libmaster.so+0x0000000c25af)
#8 kudu::master::CatalogManager::CreateTable(kudu::master::CreateTableRequestPB const*, kudu::master::CreateTableResponsePB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:505 (libmaster.so+0x0000000b36ce)
#9 kudu::master::MasterServiceImpl::CreateTable(kudu::master::CreateTableRequestPB const*, kudu::master::CreateTableResponsePB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master_service.cc:200 (libmaster.so+0x0000000fcd69)
#10 kudu::master::MasterServiceIf::Handle(kudu::rpc::InboundCall*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.service.cc:103 (libmaster_proto.so+0x0000000a1e2f)
#11 kudu::rpc::ServicePool::RunThread() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:156 (libkrpc.so+0x0000000c93ba)
#12 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000cc0ad)
#13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246 (libkrpc.so+0x0000000cc01a)
#14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkrpc.so+0x0000000cbfc3)
#15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkrpc.so+0x0000000cbdc9)
#16 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012 (libtablet.so+0x000000211501)
#17 kudu::Thread::SuperviseThread(void*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:436 (libkudu_util.so+0x00000015566b)
Mutex M345777 created at:
#0 pthread_mutex_init /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:925 (client-test+0x00000005e4ac)
#1 kudu::Mutex::Mutex() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/mutex.cc:26 (libkudu_util.so+0x000000130254)
#2 kudu::RWCLock::RWCLock() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/rwc_lock.cc:26 (libkudu_util.so+0x000000146b6d)
#3 kudu::CowObject<kudu::master::PersistentTabletInfo>::CowObject() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/cow_object.h:24 (libmaster.so+0x0000000c886a)
#4 kudu::master::TabletInfo::TabletInfo(scoped_refptr<kudu::master::TableInfo> const&, std::string const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:2134 (libmaster.so+0x0000000bef6d)
#5 kudu::master::CatalogManager::CreateTabletInfo(kudu::master::TableInfo*, std::string const&, std::string const&) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:592 (libmaster.so+0x0000000b503a)
#6 kudu::master::CatalogManager::CreateTablets(kudu::master::CreateTableRequestPB const*, kudu::master::TableInfo*, std::vector<kudu::master::TabletInfo*, std::allocator<kudu::master::TabletInfo*> >*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:562 (libmaster.so+0x0000000b444f)
#7 kudu::master::CatalogManager::CreateTable(kudu::master::CreateTableRequestPB const*, kudu::master::CreateTableResponsePB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/catalog_manager.cc:460 (libmaster.so+0x0000000b2f05)
#8 kudu::master::MasterServiceImpl::CreateTable(kudu::master::CreateTableRequestPB const*, kudu::master::CreateTableResponsePB*, kudu::rpc::RpcContext*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master_service.cc:200 (libmaster.so+0x0000000fcd69)
#9 kudu::master::MasterServiceIf::Handle(kudu::rpc::InboundCall*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.service.cc:103 (libmaster_proto.so+0x0000000a1e2f)
#10 kudu::rpc::ServicePool::RunThread() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:156 (libkrpc.so+0x0000000c93ba)
#11 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000cc0ad)
#12 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246 (libkrpc.so+0x0000000cc01a)
#13 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkrpc.so+0x0000000cbfc3)
#14 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkrpc.so+0x0000000cbdc9)
#15 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012 (libtablet.so+0x000000211501)
#16 kudu::Thread::SuperviseThread(void*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:436 (libkudu_util.so+0x00000015566b)
Thread T85 'rpc worker-2698' (tid=26987, running) created by main thread at:
#0 pthread_create /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:877 (client-test+0x00000005ddff)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, scoped_refptr<kudu::Thread>*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:366 (libkudu_util.so+0x000000155116)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::)(), kudu::rpc::ServicePool>(std::string const&, std::string const&, void (kudu::rpc::ServicePool::* const&)(), kudu::rpc::ServicePool* const&, scoped_refptr<kudu::Thread>*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.h:123 (libkrpc.so+0x0000000ca5fc)
#3 kudu::rpc::ServicePool::Init(int) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:60 (libkrpc.so+0x0000000c906d)
#4 kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/server/rpc_server.cc:82 (libserver_process.so+0x00000005aa54)
#5 kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/server/server_base.cc:171 (libserver_process.so+0x000000060710)
#6 kudu::master::Master::StartAsync() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.cc:89 (libmaster.so+0x0000000f4c02)
#7 kudu::master::MiniMaster::StartOnPorts(unsigned short, unsigned short, kudu::master::MasterOptions*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:72 (libmaster.so+0x00000010a9b6)
#8 kudu::master::MiniMaster::StartOnPorts(unsigned short, unsigned short) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:61 (libmaster.so+0x000000109e4d)
#9 kudu::master::MiniMaster::Start() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:34 (libmaster.so+0x000000109d16)
#10 kudu::MiniCluster::StartSingleMaster() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/integration-tests/mini_cluster.cc:159 (libintegration-tests.so+0x00000002a19e)
#11 kudu::MiniCluster::Start() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/integration-tests/mini_cluster.cc:74 (libintegration-tests.so+0x0000000290fb)
#12 kudu::client::ClientTest::SetUp() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/client/client-test.cc:95 (client-test+0x0000000f4989)
#13 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::)(), char const) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078 (libgmock.so+0x000000084823)
#14 main /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/test_main.cc:34:13 (libkudu_test_main.so+0x000000003b71)
Thread T22 'rpc worker-2698' (tid=26985, running) created by main thread at:
#0 pthread_create /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:877 (client-test+0x00000005ddff)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, scoped_refptr<kudu::Thread>*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:366 (libkudu_util.so+0x000000155116)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::)(), kudu::rpc::ServicePool>(std::string const&, std::string const&, void (kudu::rpc::ServicePool::* const&)(), kudu::rpc::ServicePool* const&, scoped_refptr<kudu::Thread>*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.h:123 (libkrpc.so+0x0000000ca5fc)
#3 kudu::rpc::ServicePool::Init(int) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:60 (libkrpc.so+0x0000000c906d)
#4 kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/server/rpc_server.cc:82 (libserver_process.so+0x00000005aa54)
#5 kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/server/server_base.cc:171 (libserver_process.so+0x000000060710)
#6 kudu::master::Master::StartAsync() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.cc:89 (libmaster.so+0x0000000f4c02)
#7 kudu::master::MiniMaster::StartOnPorts(unsigned short, unsigned short, kudu::master::MasterOptions*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:72 (libmaster.so+0x00000010a9b6)
#8 kudu::master::MiniMaster::StartOnPorts(unsigned short, unsigned short) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:61 (libmaster.so+0x000000109e4d)
#9 kudu::master::MiniMaster::Start() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:34 (libmaster.so+0x000000109d16)
#10 kudu::MiniCluster::StartSingleMaster() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/integration-tests/mini_cluster.cc:159 (libintegration-tests.so+0x00000002a19e)
#11 kudu::MiniCluster::Start() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/integration-tests/mini_cluster.cc:74 (libintegration-tests.so+0x0000000290fb)
#12 kudu::client::ClientTest::SetUp() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/client/client-test.cc:95 (client-test+0x0000000f4989)
#13 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::)(), char const) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078 (libgmock.so+0x000000084823)
#14 main /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/test_main.cc:34:13 (libkudu_test_main.so+0x000000003b71)
Thread T197 'rpc worker-2699' (tid=26990, running) created by main thread at:
#0 pthread_create /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/llvm-3.4.2.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:877 (client-test+0x00000005ddff)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, scoped_refptr<kudu::Thread>*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.cc:366 (libkudu_util.so+0x000000155116)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::)(), kudu::rpc::ServicePool>(std::string const&, std::string const&, void (kudu::rpc::ServicePool::* const&)(), kudu::rpc::ServicePool* const&, scoped_refptr<kudu::Thread>*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/thread.h:123 (libkrpc.so+0x0000000ca5fc)
#3 kudu::rpc::ServicePool::Init(int) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/rpc/service_pool.cc:60 (libkrpc.so+0x0000000c906d)
#4 kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/server/rpc_server.cc:82 (libserver_process.so+0x00000005aa54)
#5 kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/server/server_base.cc:171 (libserver_process.so+0x000000060710)
#6 kudu::master::Master::StartAsync() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/master.cc:89 (libmaster.so+0x0000000f4c02)
#7 kudu::master::MiniMaster::StartOnPorts(unsigned short, unsigned short, kudu::master::MasterOptions*) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:72 (libmaster.so+0x00000010a9b6)
#8 kudu::master::MiniMaster::StartOnPorts(unsigned short, unsigned short) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:61 (libmaster.so+0x000000109e4d)
#9 kudu::master::MiniMaster::Start() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/master/mini_master.cc:34 (libmaster.so+0x000000109d16)
#10 kudu::MiniCluster::StartSingleMaster() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/integration-tests/mini_cluster.cc:159 (libintegration-tests.so+0x00000002a19e)
#11 kudu::MiniCluster::Start() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/integration-tests/mini_cluster.cc:74 (libintegration-tests.so+0x0000000290fb)
#12 kudu::client::ClientTest::SetUp() /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/client/client-test.cc:95 (client-test+0x0000000f4989)
#13 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::)(), char const) /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078 (libgmock.so+0x000000084823)
#14 main /data1/jenkins-workspace/kudu-flaky-tests/BUILD_TYPE/TSAN/label/kudu-gerrit-slaves/src/kudu/util/test_main.cc:34:13 (libkudu_test_main.so+0x000000003b71)