Major Description
CBTree reports a data race in disk_reservation-itest:
WARNING: ThreadSanitizer: data race (pid=20459)
Write of size 1 at 0x7dd000002889 by thread T145:
#0 memcpy /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:677 (kudu-tserver+0x00000044cb1b)
#1 void kudu::InlineSlice<8ul, true>::set<kudu::ThreadSafeMemoryTrackingArena>(unsigned char const*, unsigned long, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:116:7 (libtablet.so+0x0000002972b6)
#2 void kudu::InlineSlice<8ul, true>::set<kudu::ThreadSafeMemoryTrackingArena>(kudu::Slice const&, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:95:5 (libtablet.so+0x000000297240)
#3 void kudu::tablet::btree::InsertInSliceArray<kudu::InlineSlice<8ul, true>, kudu::ThreadSafeMemoryTrackingArena>(kudu::InlineSlice<8ul, true>*, unsigned long, kudu::Slice const&, unsigned long, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:347:3 (libtablet.so+0x00000028bc4a)
#4 kudu::tablet::btree::InternalNode<kudu::tablet::MSBTreeTraits>::Insert(kudu::Slice const&, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:558:5 (libtablet.so+0x000000298a34)
#5 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PropagateSplitUpward(kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1524:13 (libtablet.so+0x0000002979d1)
#6 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PropagateSplitUpward(kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1556:9 (libtablet.so+0x000000297dca)
#7 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PropagateSplitUpward(kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1556:9 (libtablet.so+0x000000297dca)
#8 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::SplitLeafAndInsertUp(kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>*, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1485:5 (libtablet.so+0x000000296aeb)
#9 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::Insert(kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>*, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1312:16 (libtablet.so+0x0000002965d3)
#10 kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>::Insert(kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:861:12 (libtablet.so+0x00000028cafd)
#11 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.cc:166:27 (libtablet.so+0x00000028936b)
#12 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001dcebd)
#13 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001dde5d)
#14 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001ddcaf)
#15 kudu::tablet::WriteTransaction::Apply(gscoped_ptr<kudu::consensus::CommitMsg, kudu::DefaultDeleter<kudu::consensus::CommitMsg> >*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/write_transaction.cc:124:3 (libtablet.so+0x0000002438f1)
#16 kudu::tablet::TransactionDriver::ApplyTask() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:377:31 (libtablet.so+0x00000023b2f5)
#17 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>::Run(kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:134:12 (libtablet.so+0x00000023d2c6)
#18 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:871:5 (libtablet.so+0x00000023d255)
#19 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1063:12 (libtablet.so+0x00000023d1ea)
#20 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
#21 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
#22 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
#23 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
#24 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
#25 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#26 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
#27 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
#28 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
#29 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
#30 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
#31 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
#32 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#33 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
Previous atomic read of size 8 at 0x7dd000002888 by thread T142:
#0 __tsan_atomic64_load /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:538 (kudu-tserver+0x0000004661ac)
#1 base::subtle::Acquire_Load(long const volatile*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/atomicops-internals-tsan.h:179:10 (libtserver.so+0x00000013d2ee)
#2 kudu::InlineSlice<8ul, true>::LoadValue() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:144:27 (libtablet.so+0x0000001fbd19)
#3 kudu::InlineSlice<8ul, true>::as_slice() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:79:33 (libtablet.so+0x0000001fb826)
#4 unsigned long kudu::tablet::btree::FindInSliceArray<8ul>(kudu::InlineSlice<8ul, true> const*, long, kudu::Slice const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:330 (libtablet.so+0x0000001fb826)
#5 kudu::tablet::btree::InternalNode<kudu::tablet::MSBTreeTraits>::Find(kudu::Slice const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:576:12 (libtablet.so+0x0000001fb609)
#6 kudu::tablet::btree::InternalNode<kudu::tablet::MSBTreeTraits>::FindChild(kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:585:18 (libtablet.so+0x0000001fb3e7)
#7 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::TraverseToLeaf(kudu::Slice const&, long*) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1161:31 (libtablet.so+0x0000001fad7b)
#8 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PrepareMutation(kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1269:33 (libtablet.so+0x000000295cf2)
#9 kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>::Prepare(kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:855:5 (libtablet.so+0x00000028c6b7)
#10 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.cc:140:5 (libtablet.so+0x000000289219)
#11 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001dcebd)
#12 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001dde5d)
#13 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001ddcaf)
#14 kudu::tablet::WriteTransaction::Apply(gscoped_ptr<kudu::consensus::CommitMsg, kudu::DefaultDeleter<kudu::consensus::CommitMsg> >*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/write_transaction.cc:124:3 (libtablet.so+0x0000002438f1)
#15 kudu::tablet::TransactionDriver::ApplyTask() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:377:31 (libtablet.so+0x00000023b2f5)
#16 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>::Run(kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:134:12 (libtablet.so+0x00000023d2c6)
#17 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:871:5 (libtablet.so+0x00000023d255)
#18 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1063:12 (libtablet.so+0x00000023d1ea)
#19 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
#20 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
#21 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
#22 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
#23 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
#24 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#25 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
#26 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
#27 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
#28 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
#29 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
#30 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
#31 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#32 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
Location is heap block of size 127488 at 0x7dd000000000 allocated by thread T134:
#0 malloc /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:595 (kudu-tserver+0x000000440858)
#1 kudu::HeapBufferAllocator::Malloc(unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.cc:149:12 (libkudu_util.so+0x0000001d048f)
#2 kudu::HeapBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.cc:100:50 (libkudu_util.so+0x0000001d039a)
#3 kudu::BufferAllocator::DelegateAllocate(kudu::BufferAllocator*, unsigned long, unsigned long, kudu::BufferAllocator*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.h:211:12 (libkudu_util.so+0x0000001d11a9)
#4 kudu::MemoryTrackingBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.cc:308:22 (libkudu_util.so+0x0000001d0e5f)
#5 kudu::BufferAllocator::BestEffortAllocate(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.h:131:22 (libkudu_util.so+0x0000001cc3ab)
#6 kudu::ArenaBase<true>::NewComponent(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.cc:113:20 (libkudu_util.so+0x0000001cb1d4)
#7 kudu::ArenaBase<true>::AllocateBytesFallback(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.cc:93:26 (libkudu_util.so+0x0000001cba25)
#8 kudu::ArenaBase<true>::AllocateBytesAligned(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.h:442:10 (libtablet.so+0x000000295539)
#9 kudu::ArenaBase<true>::AllocateBytes(unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.h:141:12 (libtablet.so+0x0000002964a5)
#10 kudu::ArenaBase<true>::RelocateSlice(kudu::Slice const&, kudu::Slice*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.h:460:23 (libtablet.so+0x0000002963c3)
#11 kudu::Status kudu::RelocateIndirectDataToArena<kudu::tablet::MRSRow, kudu::ThreadSafeMemoryTrackingArena>(kudu::tablet::MRSRow*, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/common/row.h:351:12 (libtablet.so+0x0000002962c4)
#12 kudu::Status kudu::tablet::MRSRow::CopyRow<kudu::ThreadSafeMemoryTrackingArena>(kudu::ConstContiguousRow const&, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.h:135:12 (libtablet.so+0x00000028ca72)
#13 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.cc:164:31 (libtablet.so+0x00000028933f)
#14 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001dcebd)
#15 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001dde5d)
#16 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001ddcaf)
#17 kudu::tablet::WriteTransaction::Apply(gscoped_ptr<kudu::consensus::CommitMsg, kudu::DefaultDeleter<kudu::consensus::CommitMsg> >*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/write_transaction.cc:124:3 (libtablet.so+0x0000002438f1)
#18 kudu::tablet::TransactionDriver::ApplyTask() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:377:31 (libtablet.so+0x00000023b2f5)
#19 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>::Run(kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:134:12 (libtablet.so+0x00000023d2c6)
#20 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:871:5 (libtablet.so+0x00000023d255)
#21 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1063:12 (libtablet.so+0x00000023d1ea)
#22 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
#23 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
#24 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
#25 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
#26 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
#27 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#28 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
#29 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
#30 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
#31 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
#32 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
#33 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
#34 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#35 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
Thread T145 'apply [worker]-' (tid=20643, running) created by thread T124 at:
#0 pthread_create /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (kudu-tserver+0x00000044a123)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021ade7)
#2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.h:162:12 (libkudu_util.so+0x0000002262ae)
#3 kudu::ThreadPool::CreateThreadUnlocked() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:381:14 (libkudu_util.so+0x000000224304)
#4 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:211:21 (libkudu_util.so+0x000000224902)
#5 kudu::ThreadPool::SubmitFunc(boost::function<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:182:10 (libkudu_util.so+0x000000224704)
#6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:178:10 (libkudu_util.so+0x000000224632)
#7 kudu::tablet::TransactionDriver::ApplyAsync() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:358:10 (libtablet.so+0x00000023ae76)
#8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:311:31 (libtablet.so+0x00000023908d)
#9 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>::Run(kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libtablet.so+0x000000234615)
#10 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libtablet.so+0x000000234508)
#11 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1116:12 (libtablet.so+0x00000023444a)
#12 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:436:12 (libmaster_rpc.so+0x000000011c31)
#13 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus.cc:60:3 (libconsensus.so+0x000000096280)
#14 kudu::consensus::ReplicaState::AdvanceCommittedIndexUnlocked(kudu::consensus::OpId const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:624:5 (libconsensus.so+0x0000000f0f2d)
#15 kudu::consensus::ReplicaState::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:520:31 (libconsensus.so+0x0000000f0017)
#16 kudu::consensus::RaftConsensus::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:592:14 (libconsensus.so+0x0000000d76b1)
#17 kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:583:3 (libconsensus.so+0x0000000d73d0)
#18 non-virtual thunk to kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:573:21 (libconsensus.so+0x0000000d79cd)
#19 kudu::consensus::PeerMessageQueue::NotifyObserversOfMajorityReplOpChangeTask(kudu::consensus::OpId) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus_queue.cc:807:5 (libconsensus.so+0x0000000a97f2)
#20 kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>::Run(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libconsensus.so+0x0000000ba5e8)
#21 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libconsensus.so+0x0000000ba558)
#22 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId), void ()(kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>, kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1145:12 (libconsensus.so+0x0000000ba4d3)
#23 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
#24 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
#25 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
#26 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
#27 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
#28 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#29 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
#30 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
#31 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
#32 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
#33 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
#34 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
#35 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#36 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
Thread T142 'apply [worker]-' (tid=20640, running) created by thread T124 at:
#0 pthread_create /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (kudu-tserver+0x00000044a123)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021ade7)
#2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.h:162:12 (libkudu_util.so+0x0000002262ae)
#3 kudu::ThreadPool::CreateThreadUnlocked() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:381:14 (libkudu_util.so+0x000000224304)
#4 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:211:21 (libkudu_util.so+0x000000224902)
#5 kudu::ThreadPool::SubmitFunc(boost::function<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:182:10 (libkudu_util.so+0x000000224704)
#6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:178:10 (libkudu_util.so+0x000000224632)
#7 kudu::tablet::TransactionDriver::ApplyAsync() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:358:10 (libtablet.so+0x00000023ae76)
#8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:311:31 (libtablet.so+0x00000023908d)
#9 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>::Run(kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libtablet.so+0x000000234615)
#10 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libtablet.so+0x000000234508)
#11 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1116:12 (libtablet.so+0x00000023444a)
#12 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:436:12 (libmaster_rpc.so+0x000000011c31)
#13 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus.cc:60:3 (libconsensus.so+0x000000096280)
#14 kudu::consensus::ReplicaState::AdvanceCommittedIndexUnlocked(kudu::consensus::OpId const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:624:5 (libconsensus.so+0x0000000f0f2d)
#15 kudu::consensus::ReplicaState::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:520:31 (libconsensus.so+0x0000000f0017)
#16 kudu::consensus::RaftConsensus::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:592:14 (libconsensus.so+0x0000000d76b1)
#17 kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:583:3 (libconsensus.so+0x0000000d73d0)
#18 non-virtual thunk to kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:573:21 (libconsensus.so+0x0000000d79cd)
#19 kudu::consensus::PeerMessageQueue::NotifyObserversOfMajorityReplOpChangeTask(kudu::consensus::OpId) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus_queue.cc:807:5 (libconsensus.so+0x0000000a97f2)
#20 kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>::Run(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libconsensus.so+0x0000000ba5e8)
#21 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libconsensus.so+0x0000000ba558)
#22 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId), void ()(kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>, kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1145:12 (libconsensus.so+0x0000000ba4d3)
#23 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
#24 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
#25 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
#26 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
#27 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
#28 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#29 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
#30 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
#31 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
#32 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
#33 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
#34 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
#35 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#36 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
Thread T134 'apply [worker]-' (tid=20632, running) created by thread T124 at:
#0 pthread_create /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (kudu-tserver+0x00000044a123)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021ade7)
#2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.h:162:12 (libkudu_util.so+0x0000002262ae)
#3 kudu::ThreadPool::CreateThreadUnlocked() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:381:14 (libkudu_util.so+0x000000224304)
#4 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:211:21 (libkudu_util.so+0x000000224902)
#5 kudu::ThreadPool::SubmitFunc(boost::function<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:182:10 (libkudu_util.so+0x000000224704)
#6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:178:10 (libkudu_util.so+0x000000224632)
#7 kudu::tablet::TransactionDriver::ApplyAsync() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:358:10 (libtablet.so+0x00000023ae76)
#8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:311:31 (libtablet.so+0x00000023908d)
#9 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>::Run(kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libtablet.so+0x000000234615)
#10 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libtablet.so+0x000000234508)
#11 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1116:12 (libtablet.so+0x00000023444a)
#12 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:436:12 (libmaster_rpc.so+0x000000011c31)
#13 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus.cc:60:3 (libconsensus.so+0x000000096280)
#14 kudu::consensus::ReplicaState::AdvanceCommittedIndexUnlocked(kudu::consensus::OpId const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:624:5 (libconsensus.so+0x0000000f0f2d)
#15 kudu::consensus::ReplicaState::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:520:31 (libconsensus.so+0x0000000f0017)
#16 kudu::consensus::RaftConsensus::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:592:14 (libconsensus.so+0x0000000d76b1)
#17 kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:583:3 (libconsensus.so+0x0000000d73d0)
#18 non-virtual thunk to kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:573:21 (libconsensus.so+0x0000000d79cd)
#19 kudu::consensus::PeerMessageQueue::NotifyObserversOfMajorityReplOpChangeTask(kudu::consensus::OpId) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus_queue.cc:807:5 (libconsensus.so+0x0000000a97f2)
#20 kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>::Run(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libconsensus.so+0x0000000ba5e8)
#21 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libconsensus.so+0x0000000ba558)
#22 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId), void ()(kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>, kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1145:12 (libconsensus.so+0x0000000ba4d3)
#23 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
#24 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
#25 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
#26 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
#27 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
#28 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#29 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
#30 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
#31 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
#32 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
#33 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
#34 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
#35 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
#36 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
SUMMARY: ThreadSanitizer: data race /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:116:7 in void kudu::InlineSlice<8ul, true>::set<kudu::ThreadSafeMemoryTrackingArena>(unsigned char const*, unsigned long, kudu::ThreadSafeMemoryTrackingArena*)