Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Duplicate
-
None
-
None
-
None
Description
This came up in a dist-test run of a TSAN build on TabletReplacementITest.TestRemoteBoostrapWithPendingConfigChangeCommits as part of KUDU-1233. I am somewhat skeptical that it's a real bug, but it could just be somehow misreported by TSAN and the real error is elsewhere.
WARNING: ThreadSanitizer: data race (pid=20565)
Write of size 8 at 0x7d1400027e88 by thread T124:
#0 operator delete(void*) <null> (kudu-tserver+0x0000000494e6)
#1 std::string::assign(std::string const&) <null> (libstdc++.so.6+0x0000000bb4e8)
#2 kudu::tablet::TabletBootstrap::Bootstrap(std::tr1::shared_ptr<kudu::tablet::Tablet>*, scoped_refptr<kudu::log::Log>*, kudu::consensus::ConsensusBootstrapInfo*) /home/todd/git/kudu/src/kudu/tablet/tablet_bootstrap.cc:439 (libtablet.so+0x000000240bd8)
#3 kudu::tablet::BootstrapTablet(scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::server::Clock> const&, std::tr1::shared_ptr<kudu::MemTracker> const&, kudu::MetricRegistry*, kudu::tablet::TabletStatusListener*, std::tr1::shared_ptr<kudu::tablet::Tablet>*, scoped_refptr<kudu::log::Log>*, scoped_refptr<kudu::log::LogAnchorRegistry> const&, kudu::consensus::ConsensusBootstrapInfo*) /home/todd/git/kudu/src/kudu/tablet/tablet_bootstrap.cc:376 (libtablet.so+0x000000240952)
#4 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&) /home/todd/git/kudu/src/kudu/tserver/ts_tablet_manager.cc:607 (libtserver.so+0x000000161d4e)
#5 boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>::operator()(kudu::tserver::TSTabletManager*, scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&) const /usr/include/boost/bind/mem_fn_template.hpp:280 (libtserver.so+0x00000016e05e)
#6 void boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>, boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >, boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > >::operator()<boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:392 (libtserver.so+0x00000016dfb3)
#7 boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>, boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>, boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >, boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libtserver.so+0x00000016df33)
#8 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::tserver::TSTabletManager, scoped_refptr<kudu::tablet::TabletMetadata> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&>, boost::_bi::list3<boost::_bi::value<kudu::tserver::TSTabletManager*>, boost::_bi::value<scoped_refptr<kudu::tablet::TabletMetadata> >, boost::_bi::value<scoped_refptr<kudu::tserver::TransitionInProgressDeleter> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libtserver.so+0x00000016dcf1)
#9 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766 (libkrpc.so+0x000000096a61)
#10 kudu::FunctionRunnable::Run() /home/todd/git/kudu/src/kudu/util/threadpool.cc:46 (libkudu_util.so+0x00000021099d)
#11 kudu::ThreadPool::DispatchThread(bool) /home/todd/git/kudu/src/kudu/util/threadpool.cc:317 (libkudu_util.so+0x00000020f4b6)
#12 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:165 (libkudu_util.so+0x000000212845)
#13 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313 (libkudu_util.so+0x0000002127ab)
#14 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkudu_util.so+0x000000212733)
#15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkudu_util.so+0x000000212541)
#16 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766 (libkrpc.so+0x000000096a61)
#17 kudu::Thread::SuperviseThread(void*) /home/todd/git/kudu/src/kudu/util/thread.cc:503 (libkudu_util.so+0x0000002052b9)
Previous read of size 8 at 0x7d1400027e88 by thread T12:
#0 memcpy <null> (kudu-tserver+0x0000000509fb)
#1 std::string::data() const /home/todd/git/kudu/thirdparty/protobuf-2.6.1/src/google/protobuf/io/coded_stream.cc:640 (libprotobuf.so.9+0x000000072a4c)
#2 unsigned char* google::protobuf::internal::WireFormatLite::WriteMessageNoVirtualToArray<kudu::tablet::TabletStatusPB>(int, kudu::tablet::TabletStatusPB const&, unsigned char*) /home/todd/git/kudu/thirdparty/installed/include/google/protobuf/wire_format_lite_inl.h:793 (libtserver_proto.so+0x00000002eb70)
#3 unsigned char* google::protobuf::internal::WireFormatLite::WriteMessageNoVirtualToArray<kudu::tserver::ListTabletsResponsePB_StatusAndSchemaPB>(int, kudu::tserver::ListTabletsResponsePB_StatusAndSchemaPB const&, unsigned char*) /home/todd/git/kudu/thirdparty/installed/include/google/protobuf/wire_format_lite_inl.h:793 (libtserver_proto.so+0x00000002faf9)
#4 kudu::rpc::serialization::SerializeMessage(google::protobuf::MessageLite const&, kudu::faststring*, int, bool) /home/todd/git/kudu/src/kudu/rpc/serialization.cc:65 (libkrpc.so+0x0000000d8841)
#5 kudu::rpc::InboundCall::SerializeResponseBuffer(google::protobuf::MessageLite const&, bool) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:144 (libkrpc.so+0x0000000a7fb2)
#6 kudu::rpc::InboundCall::Respond(google::protobuf::MessageLite const&, bool) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:116 (libkrpc.so+0x0000000a7754)
#7 kudu::rpc::InboundCall::RespondSuccess(google::protobuf::MessageLite const&) /home/todd/git/kudu/src/kudu/rpc/inbound_call.cc:79 (libkrpc.so+0x0000000a7665)
#8 kudu::rpc::RpcContext::RespondSuccess() /home/todd/git/kudu/src/kudu/rpc/rpc_context.cc:51 (libkrpc.so+0x0000000c727d)
#9 kudu::tserver::TabletServiceImpl::ListTablets(kudu::tserver::ListTabletsRequestPB const*, kudu::tserver::ListTabletsResponsePB*, kudu::rpc::RpcContext*) /home/todd/git/kudu/src/kudu/tserver/tablet_service.cc:1078 (libtserver.so+0x00000014d325)
#10 kudu::tserver::TabletServerServiceIf::Handle(kudu::rpc::InboundCall*) /home/todd/git/kudu/src/kudu/tserver/tserver_service.service.cc:122 (libtserver_service_proto.so+0x000000015a56)
#11 kudu::rpc::ServicePool::RunThread() /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:173 (libkrpc.so+0x0000000dac85)
#12 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000dcf9d)
#13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:253 (libkrpc.so+0x0000000dcf0a)
#14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkrpc.so+0x0000000dceb3)
#15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkrpc.so+0x0000000dccd9)
#16 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766 (libkrpc.so+0x000000096a61)
#17 kudu::Thread::SuperviseThread(void*) /home/todd/git/kudu/src/kudu/util/thread.cc:503 (libkudu_util.so+0x0000002052b9)
Thread T124 'tablet-bootstra' (tid=20737, running) created by thread T64 at:
#0 pthread_create <null> (kudu-tserver+0x00000004e7f3)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.cc:431 (libkudu_util.so+0x000000204d86)
#2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.h:143 (libkudu_util.so+0x00000021067b)
#3 kudu::ThreadPool::CreateThreadUnlocked() /home/todd/git/kudu/src/kudu/util/threadpool.cc:344 (libkudu_util.so+0x00000020e983)
#4 kudu::ThreadPool::Submit(std::tr1::shared_ptr<kudu::Runnable> const&) /home/todd/git/kudu/src/kudu/util/threadpool.cc:192 (libkudu_util.so+0x00000020ee42)
#5 kudu::ThreadPool::SubmitFunc(boost::function<void ()> const&) /home/todd/git/kudu/src/kudu/util/threadpool.cc:163 (libkudu_util.so+0x00000020ec55)
#6 kudu::tserver::TSTabletManager::StartRemoteBootstrap(kudu::consensus::StartRemoteBootstrapRequestPB const&) /home/todd/git/kudu/src/kudu/tserver/ts_tablet_manager.cc:444 (libtserver.so+0x000000163e3c)
#7 kudu::tserver::ConsensusServiceImpl::StartRemoteBootstrap(kudu::consensus::StartRemoteBootstrapRequestPB const*, kudu::consensus::StartRemoteBootstrapResponsePB*, kudu::rpc::RpcContext*) /home/todd/git/kudu/src/kudu/tserver/tablet_service.cc:951 (libtserver.so+0x00000014959b)
#8 kudu::consensus::ConsensusServiceIf::Handle(kudu::rpc::InboundCall*) /home/todd/git/kudu/src/kudu/consensus/consensus.service.cc:192 (libconsensus_proto.so+0x0000000680cf)
#9 kudu::rpc::ServicePool::RunThread() /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:173 (libkrpc.so+0x0000000dac85)
#10 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /usr/include/boost/bind/mem_fn_template.hpp:49 (libkrpc.so+0x0000000dcf9d)
#11 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:253 (libkrpc.so+0x0000000dcf0a)
#12 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20 (libkrpc.so+0x0000000dceb3)
#13 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153 (libkrpc.so+0x0000000dccd9)
#14 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766 (libkrpc.so+0x000000096a61)
#15 kudu::Thread::SuperviseThread(void*) /home/todd/git/kudu/src/kudu/util/thread.cc:503 (libkudu_util.so+0x0000002052b9)
Thread T12 'rpc worker-2058' (tid=20580, running) created by main thread at:
#0 pthread_create <null> (kudu-tserver+0x00000004e7f3)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.cc:431 (libkudu_util.so+0x000000204d86)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ServicePool::*)(), kudu::rpc::ServicePool*>(std::string const&, std::string const&, void (kudu::rpc::ServicePool::* const&)(), kudu::rpc::ServicePool* const&, scoped_refptr<kudu::Thread>*) /home/todd/git/kudu/src/kudu/util/thread.h:137 (libkrpc.so+0x0000000dbbeb)
#3 kudu::rpc::ServicePool::Init(int) /home/todd/git/kudu/src/kudu/rpc/service_pool.cc:77 (libkrpc.so+0x0000000da994)
#4 kudu::RpcServer::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /home/todd/git/kudu/src/kudu/server/rpc_server.cc:122 (libserver_process.so+0x00000007f2c6)
#5 kudu::server::ServerBase::RegisterService(gscoped_ptr<kudu::rpc::ServiceIf, kudu::DefaultDeleter<kudu::rpc::ServiceIf> >) /home/todd/git/kudu/src/kudu/server/server_base.cc:237 (libserver_process.so+0x000000086160)
#6 kudu::tserver::TabletServer::Start() /home/todd/git/kudu/src/kudu/tserver/tablet_server.cc:117 (libtserver.so+0x00000013ee50)
#7 kudu::tserver::TabletServerMain(int, char**) /home/todd/git/kudu/src/kudu/tserver/tablet_server_main.cc:55 (kudu-tserver+0x0000000a3bcc)
#8 main /home/todd/git/kudu/src/kudu/tserver/tablet_server_main.cc:69 (kudu-tserver+0x0000000a39ae)
SUMMARY: ThreadSanitizer: data race (/tmp/run_tha_testcGwYR3/build/fastdebug/kudu-tserver+0x494e6) in operator delete(void*)