Major Description
Saw a TSAN error on a Jenkins run: http://104.196.14.100/job/kudu-gerrit/1936/BUILD_TYPE=TSAN/testReport/junit/(root)/MultiThreadedTabletTest_0/DeleteAndReinsert/
MultiThreadedTabletTest/0.DeleteAndReinsert (from MultiThreadedTabletTest_0)
WARNING: ThreadSanitizer: data race (pid=8654) Atomic write of size 8 at 0x7d900002e550 by thread T14:
#0 __tsan_atomic64_store /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:565 (mt-tablet-test+0x0000004a2e03)
#1 base::subtle::Release_Store(long volatile*, long) /home/jenkins-slave/workspace/kudu-1/src/kudu/gutil/atomicops-internals-tsan.h:171:3 (libtablet.so+0x000000232ec5)
#2 kudu::tablet::Mutation::AppendToListAtomic(kudu::tablet::Mutation**) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mutation.cc:60:5 (libtablet.so+0x0000002a997f)
#3 kudu::tablet::MemRowSet::MutateRow(kudu::Timestamp, kudu::tablet::RowSetKeyProbe const&, kudu::RowChangeList const&, kudu::consensus::OpId const&, kudu::tablet::ProbeStats*, kudu::tablet::OperationResultPB*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/memrowset.cc:234:5 (libtablet.so+0x00000029566d)
#4 kudu::tablet::DuplicatingRowSet::MutateRow(kudu::Timestamp, kudu::tablet::RowSetKeyProbe const&, kudu::RowChangeList const&, kudu::consensus::OpId const&, kudu::tablet::ProbeStats*, kudu::tablet::OperationResultPB*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/rowset.cc:120:16 (libtablet.so+0x0000002b3fc6)
#5 kudu::tablet::Tablet::MutateRowUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:556:9 (libtablet.so+0x0000001e6a0e)
#6 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:623:21 (libtablet.so+0x0000001e6f2b)
#7 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001e6d5f)
#8 kudu::tablet::LocalTabletWriter::WriteBatch(std::vector<kudu::tablet::LocalTabletWriter::Op, std::allocator<kudu::tablet::LocalTabletWriter::Op> > const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:103:5 (mt-tablet-test+0x0000004f9bea)
#9 kudu::tablet::LocalTabletWriter::Write(kudu::RowOperationsPB_Type, kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:84:12 (mt-tablet-test+0x0000004f98bd)
#10 kudu::tablet::LocalTabletWriter::Delete(kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:71:12 (mt-tablet-test+0x00000051de6e)
#11 kudu::tablet::TabletTestBase<kudu::tablet::StringKeyTestSetup>::DeleteTestRow(kudu::tablet::LocalTabletWriter*, long) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet-test-base.h:397:12 (mt-tablet-test+0x00000051de09)
#12 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::DeleteAndReinsertCycleThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:331:9 (mt-tablet-test+0x00000051d878)
#13 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed)
#14 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a)
#15 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3)
#16 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1)
#17 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561)
#18 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af)
Previous read of size 8 at 0x7d900002e550 by thread T4 (mutexes: write M15465):
#0 kudu::tablet::Mutation::next() const /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mutation.h:54:41 (libtablet.so+0x000000268a0a)
#1 kudu::tablet::ReupdateMissedDeltas(std::string const&, kudu::tablet::CompactionInput*, kudu::tablet::MvccSnapshot const&, kudu::tablet::MvccSnapshot const&, std::vector<std::shared_ptr<kudu::tablet::RowSet>, std::allocator<std::shared_ptr<kudu::tablet::RowSet> > > const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/compaction.cc:886:18 (libtablet.so+0x0000002625c2)
#2 kudu::tablet::Tablet::DoCompactionOrFlush(kudu::tablet::RowSetsInCompaction const&, long) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:1385:3 (libtablet.so+0x0000001e9028)
#3 kudu::tablet::Tablet::FlushInternal(kudu::tablet::RowSetsInCompaction const&, std::shared_ptr<kudu::tablet::MemRowSet> const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:777:3 (libtablet.so+0x0000001e80a5)
#4 kudu::tablet::Tablet::FlushUnlocked() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:712:10 (libtablet.so+0x0000001e79ae)
#5 kudu::tablet::Tablet::Flush() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:694:10 (libtablet.so+0x0000001e77e0)
#6 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::FlushThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:269:9 (mt-tablet-test+0x0000004f461d)
#7 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed)
#8 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a)
#9 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3)
#10 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1)
#11 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561)
#12 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af)
Location is heap block of size 7968 at 0x7d900002e000 allocated by thread T19:
#0 malloc /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:595 (mt-tablet-test+0x00000047d16f)
#1 kudu::HeapBufferAllocator::Malloc(unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.cc:149:12 (libkudu_util.so+0x0000001d2fef)
#2 kudu::HeapBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.cc:100:50 (libkudu_util.so+0x0000001d2efa)
#3 kudu::BufferAllocator::DelegateAllocate(kudu::BufferAllocator*, unsigned long, unsigned long, kudu::BufferAllocator*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.h:211:12 (libkudu_util.so+0x0000001d3d09)
#4 kudu::MemoryTrackingBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.cc:308:22 (libkudu_util.so+0x0000001d39bf)
#5 kudu::BufferAllocator::BestEffortAllocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.h:131:22 (libkudu_util.so+0x0000001cef0b)
#6 kudu::ArenaBase<true>::NewComponent(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/arena.cc:113:20 (libkudu_util.so+0x0000001cdd34)
#7 kudu::ArenaBase<true>::AllocateBytesFallback(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/arena.cc:93:26 (libkudu_util.so+0x0000001ce585)
#8 kudu::ArenaBase<true>::AllocateBytesAligned(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/arena.h:442:10 (libtablet.so+0x0000002a11e9)
#9 kudu::tablet::Mutation* kudu::tablet::Mutation::CreateInArena<kudu::ThreadSafeMemoryTrackingArena>(kudu::ThreadSafeMemoryTrackingArena*, kudu::Timestamp, kudu::RowChangeList const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mutation.h:115:19 (libtablet.so+0x000000298934)
#10 kudu::tablet::MemRowSet::Reinsert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::tablet::MRSRow*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/memrowset.cc:195:19 (libtablet.so+0x0000002953cb)
#11 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/memrowset.cc:156:14 (libtablet.so+0x000000294f39)
#12 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001e5f6d)
#13 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001e6f0d)
#14 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001e6d5f)
#15 kudu::tablet::LocalTabletWriter::WriteBatch(std::vector<kudu::tablet::LocalTabletWriter::Op, std::allocator<kudu::tablet::LocalTabletWriter::Op> > const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:103:5 (mt-tablet-test+0x0000004f9bea)
#16 kudu::tablet::LocalTabletWriter::Write(kudu::RowOperationsPB_Type, kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:84:12 (mt-tablet-test+0x0000004f98bd)
#17 kudu::tablet::LocalTabletWriter::Insert(kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:63:12 (mt-tablet-test+0x0000004f928e)
#18 kudu::tablet::TabletTestBase<kudu::tablet::StringKeyTestSetup>::InsertTestRow(kudu::tablet::LocalTabletWriter*, long, int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet-test-base.h:366:12 (mt-tablet-test+0x00000051dd71)
#19 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::DeleteAndReinsertCycleThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:330:9 (mt-tablet-test+0x00000051d84f)
#20 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed)
#21 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a)
#22 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3)
#23 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1)
#24 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af)
Mutex M15465 (0x7d500001c958) created at:
#0 pthread_mutex_trylock /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1194 (mt-tablet-test+0x000000464f53)
#1 __gthread_mutex_trylock(pthread_mutex_t*) /home/jenkins-slave/workspace/kudu-1/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/gthr-default.h:757:12 (libtablet.so+0x0000001efee6)
#2 std::mutex::try_lock() /home/jenkins-slave/workspace/kudu-1/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/mutex:146:15 (libtablet.so+0x000000205d59)
#3 std::unique_lock<std::mutex>::unique_lock(std::mutex&, std::try_to_lock_t) /home/jenkins-slave/workspace/kudu-1/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/mutex:415:34 (libtablet.so+0x0000001f4dcf)
#4 kudu::tablet::Tablet::ReplaceMemRowSetUnlocked(kudu::tablet::RowSetsInCompaction*, std::shared_ptr<kudu::tablet::MemRowSet>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:720:32 (libtablet.so+0x0000001e7af7)
#5 kudu::tablet::Tablet::FlushUnlocked() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:704:5 (libtablet.so+0x0000001e7972)
#6 kudu::tablet::Tablet::Flush() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:694:10 (libtablet.so+0x0000001e77e0)
#7 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::FlushThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:269:9 (mt-tablet-test+0x0000004f461d)
#8 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed)
#9 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a)
#10 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3)
#11 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1)
#12 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561)
#13 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af)
Thread T14 'test4-8671' (tid=8671, running) created by main thread at:
#0 pthread_create /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (mt-tablet-test+0x000000486403)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021de37)
#2 kudu::Status kudu::Thread::Create<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int>(std::string const&, std::string const&, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>* const&, int const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:162:12 (mt-tablet-test+0x0000004f727d)
#3 void kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::StartThreads<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int)>(int, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int)) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:376:7 (mt-tablet-test+0x0000004f3d9a)
#4 kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::StringKeyTestSetup>::TestBody() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:451:3 (mt-tablet-test+0x00000051d642)
#5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10 (libgmock.so+0x0000000442f9)
#6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114 (libgmock.so+0x0000000442f9)
#7 main /home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:48:13 (libkudu_test_main.so+0x000000001e75)
Thread T4 'test0-8661' (tid=8661, running) created by main thread at:
#0 pthread_create /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (mt-tablet-test+0x000000486403)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021de37)
#2 kudu::Status kudu::Thread::Create<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int>(std::string const&, std::string const&, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>* const&, int const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:162:12 (mt-tablet-test+0x0000004f727d)
#3 void kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::StartThreads<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int)>(int, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int)) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:376:7 (mt-tablet-test+0x0000004f3d9a)
#4 kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::StringKeyTestSetup>::TestBody() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:444:3 (mt-tablet-test+0x00000051d558)
#5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10 (libgmock.so+0x0000000442f9)
#6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114 (libgmock.so+0x0000000442f9)
#7 main /home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:48:13 (libkudu_test_main.so+0x000000001e75)
Thread T19 'test9-8677' (tid=8677, running) created by main thread at:
#0 pthread_create /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (mt-tablet-test+0x000000486403)
#1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021de37)
#2 kudu::Status kudu::Thread::Create<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int>(std::string const&, std::string const&, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>* const&, int const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:162:12 (mt-tablet-test+0x0000004f727d)
#3 void kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::StartThreads<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int)>(int, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int)) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:376:7 (mt-tablet-test+0x0000004f3d9a)
#4 kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::StringKeyTestSetup>::TestBody() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:451:3 (mt-tablet-test+0x00000051d642)
#5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10 (libgmock.so+0x0000000442f9)
#6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114 (libgmock.so+0x0000000442f9)
#7 main /home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:48:13 (libkudu_test_main.so+0x000000001e75)