[KUDU-2480] tsan failure of master-stress-test

• Type: Test
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 1.8.0
• Assignee: Dan Burkert
• Reporter: Hao Hao
• Created: 19/Jun/18 18:11
• Updated: 20/May/19 21:58
• Resolved: 20/May/19 21:58

Description

master-stress-test recently has been very flaky(~24%).  One of the failure log

WARNING: ThreadSanitizer: data race (pid=26513)
 Read of size 8 at 0x7ffb5e5b88b8 by thread T65:
 #0 kudu::Status::operator=(kudu::Status const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/status.h:469:7 (libmaster.so+0x10bd00)
 #1 kudu::Synchronizer::StatusCB(kudu::Status const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/async_util.h:44:8 (libmaster.so+0x10bc40)
 #2 kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status const&)>::Run(kudu::Synchronizer*, kudu::Status const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:192:12 (libmaster.so+0x10c708)
 #3 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status const&)>, void ()(kudu::Synchronizer*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status const&)>, kudu::Synchronizer*, kudu::Status const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:889:14 (libmaster.so+0x10c5e8)
 #4 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::Synchronizer::*)(kudu::Status const&)>, void ()(kudu::Synchronizer*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::Synchronizer>)>, void ()(kudu::Synchronizer*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/bind_internal.h:1118:12 (libmaster.so+0x10c51a)
 #5 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:436:12 (libmaster.so+0x10b831)
 #6 kudu::master::HmsNotificationLogListenerTask::RunLoop() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/hms_notification_log_listener.cc:136:10 (libmaster.so+0x108e0a)
 #7 boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>::operator()(kudu::master::HmsNotificationLogListenerTask*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libmaster.so+0x110ea9)
 #8 void boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*> >::operator()<boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libmaster.so+0x110dfa)
 #9 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>, boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libmaster.so+0x110d83)
 #10 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::master::HmsNotificationLogListenerTask>, boost::_bi::list1<boost::_bi::value<kudu::master::HmsNotificationLogListenerTask*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libmaster.so+0x110b79)
 #11 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb64b1)
 #12 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:603:3 (libkudu_util.so+0x1bd8b4)

Previous write of size 8 at 0x7ffb5e5b88b8 by thread T24 (mutexes: read M1468):
 #0 boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::init(boost::intrusive::list_node<void*>* const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/circular_list_algorithms.hpp:72:22 (libkrpc.so+0x99c92)
 #1 boost::intrusive::generic_hook<boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >, boost::intrusive::dft_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)1>::generic_hook() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/detail/generic_hook.hpp:174:10 (libkrpc.so+0xc4669)
 #2 boost::intrusive::list_base_hook<void, void, void>::list_base_hook() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/intrusive/list_hook.hpp:83:7 (libkrpc.so+0xc2049)
 #3 kudu::rpc::ReactorTask::ReactorTask() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:262:14 (libkrpc.so+0xbd5fb)
 #4 kudu::rpc::QueueTransferTask::QueueTransferTask(gscoped_ptr<kudu::rpc::OutboundTransfer, kudu::DefaultDeleter<kudu::rpc::OutboundTransfer> >, kudu::rpc::Connection*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/connection.cc:432:3 (libkrpc.so+0x98ea4)
 #5 kudu::rpc::Connection::QueueResponseForCall(gscoped_ptr<kudu::rpc::InboundCall, kudu::DefaultDeleter<kudu::rpc::InboundCall> >) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/connection.cc:474:33 (libkrpc.so+0x94d79)
 #6 kudu::rpc::InboundCall::Respond(google::protobuf::MessageLite const&, bool) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/inbound_call.cc:165:10 (libkrpc.so+0xa11b9)
 #7 kudu::rpc::InboundCall::RespondSuccess(google::protobuf::MessageLite const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/inbound_call.cc:109:3 (libkrpc.so+0xa0fd5)
 #8 kudu::rpc::RpcContext::RespondSuccess() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/rpc_context.cc:77:12 (libkrpc.so+0xd5b35)
 #9 kudu::master::MasterServiceImpl::AlterTable(kudu::master::AlterTableRequestPB const*, kudu::master::AlterTableResponsePB*, kudu::rpc::RpcContext*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_service.cc:350:8 (libmaster.so+0x128903)
 #10 kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/master/master.service.cc:259:13 (libmaster_proto.so+0xe2544)
 #11 _ZNSt3__18__invokeIRZN4kudu6master15MasterServiceIfC1ERK13scoped_refptrINS1_12MetricEntityEERKS4_INS1_3rpc13ResultTrackerEEE4$_13JPKN6google8protobuf7MessageEPSI_PNS9_10RpcContextEEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSO_DpOSP_ /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 (libmaster_proto.so+0xe24d1)
 #12 _ZNSt3__128__invoke_void_return_wrapperIvE6__callIJRZN4kudu6master15MasterServiceIfC1ERK13scoped_refptrINS3_12MetricEntityEERKS6_INS3_3rpc13ResultTrackerEEE4$_13PKN6google8protobuf7MessageEPSK_PNSB_10RpcContextEEEEvDpOT_ /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:349 (libmaster_proto.so+0xe24d1)
 #13 std::__1::__function::__func<kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13, std::__1::allocator<kudu::master::MasterServiceIf::MasterServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_13>, void ()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 (libmaster_proto.so+0xe23e4)
 #14 std::__1::function<void ()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 (libkrpc.so+0xefce9)
 #15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_if.cc:145:3 (libkrpc.so+0xef632)
 #16 kudu::rpc::ServicePool::RunThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:225:15 (libkrpc.so+0xf0b8c)
 #17 boost::_mfi::mf0<void, kudu::rpc::ServicePool>::operator()(kudu::rpc::ServicePool*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xf2a69)
 #18 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ServicePool>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xf29ba)
 #19 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xf2943)
 #20 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ServicePool>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ServicePool*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xf2739)
 #21 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb64b1)
 #22 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:603:3 (libkudu_util.so+0x1bd8b4)

Location is stack of thread T24.