Description
Saw a TSAN error on a Jenkins run: http://104.196.14.100/job/kudu-gerrit/1936/BUILD_TYPE=TSAN/testReport/junit/(root)/MultiThreadedTabletTest_0/DeleteAndReinsert/
MultiThreadedTabletTest/0.DeleteAndReinsert (from MultiThreadedTabletTest_0)
WARNING: ThreadSanitizer: data race (pid=8654) Atomic write of size 8 at 0x7d900002e550 by thread T14: #0 __tsan_atomic64_store /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:565 (mt-tablet-test+0x0000004a2e03) #1 base::subtle::Release_Store(long volatile*, long) /home/jenkins-slave/workspace/kudu-1/src/kudu/gutil/atomicops-internals-tsan.h:171:3 (libtablet.so+0x000000232ec5) #2 kudu::tablet::Mutation::AppendToListAtomic(kudu::tablet::Mutation**) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mutation.cc:60:5 (libtablet.so+0x0000002a997f) #3 kudu::tablet::MemRowSet::MutateRow(kudu::Timestamp, kudu::tablet::RowSetKeyProbe const&, kudu::RowChangeList const&, kudu::consensus::OpId const&, kudu::tablet::ProbeStats*, kudu::tablet::OperationResultPB*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/memrowset.cc:234:5 (libtablet.so+0x00000029566d) #4 kudu::tablet::DuplicatingRowSet::MutateRow(kudu::Timestamp, kudu::tablet::RowSetKeyProbe const&, kudu::RowChangeList const&, kudu::consensus::OpId const&, kudu::tablet::ProbeStats*, kudu::tablet::OperationResultPB*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/rowset.cc:120:16 (libtablet.so+0x0000002b3fc6) #5 kudu::tablet::Tablet::MutateRowUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:556:9 (libtablet.so+0x0000001e6a0e) #6 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:623:21 (libtablet.so+0x0000001e6f2b) #7 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001e6d5f) #8 kudu::tablet::LocalTabletWriter::WriteBatch(std::vector<kudu::tablet::LocalTabletWriter::Op, std::allocator<kudu::tablet::LocalTabletWriter::Op> > const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:103:5 (mt-tablet-test+0x0000004f9bea) #9 kudu::tablet::LocalTabletWriter::Write(kudu::RowOperationsPB_Type, kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:84:12 (mt-tablet-test+0x0000004f98bd) #10 kudu::tablet::LocalTabletWriter::Delete(kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:71:12 (mt-tablet-test+0x00000051de6e) #11 kudu::tablet::TabletTestBase<kudu::tablet::StringKeyTestSetup>::DeleteTestRow(kudu::tablet::LocalTabletWriter*, long) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet-test-base.h:397:12 (mt-tablet-test+0x00000051de09) #12 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::DeleteAndReinsertCycleThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:331:9 (mt-tablet-test+0x00000051d878) #13 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed) #14 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a) #15 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3) #16 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1) #17 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561) #18 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af) Previous read of size 8 at 0x7d900002e550 by thread T4 (mutexes: write M15465): #0 kudu::tablet::Mutation::next() const /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mutation.h:54:41 (libtablet.so+0x000000268a0a) #1 kudu::tablet::ReupdateMissedDeltas(std::string const&, kudu::tablet::CompactionInput*, kudu::tablet::MvccSnapshot const&, kudu::tablet::MvccSnapshot const&, std::vector<std::shared_ptr<kudu::tablet::RowSet>, std::allocator<std::shared_ptr<kudu::tablet::RowSet> > > const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/compaction.cc:886:18 (libtablet.so+0x0000002625c2) #2 kudu::tablet::Tablet::DoCompactionOrFlush(kudu::tablet::RowSetsInCompaction const&, long) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:1385:3 (libtablet.so+0x0000001e9028) #3 kudu::tablet::Tablet::FlushInternal(kudu::tablet::RowSetsInCompaction const&, std::shared_ptr<kudu::tablet::MemRowSet> const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:777:3 (libtablet.so+0x0000001e80a5) #4 kudu::tablet::Tablet::FlushUnlocked() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:712:10 (libtablet.so+0x0000001e79ae) #5 kudu::tablet::Tablet::Flush() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:694:10 (libtablet.so+0x0000001e77e0) #6 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::FlushThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:269:9 (mt-tablet-test+0x0000004f461d) #7 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed) #8 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a) #9 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3) #10 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1) #11 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561) #12 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af) Location is heap block of size 7968 at 0x7d900002e000 allocated by thread T19: #0 malloc /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:595 (mt-tablet-test+0x00000047d16f) #1 kudu::HeapBufferAllocator::Malloc(unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.cc:149:12 (libkudu_util.so+0x0000001d2fef) #2 kudu::HeapBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.cc:100:50 (libkudu_util.so+0x0000001d2efa) #3 kudu::BufferAllocator::DelegateAllocate(kudu::BufferAllocator*, unsigned long, unsigned long, kudu::BufferAllocator*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.h:211:12 (libkudu_util.so+0x0000001d3d09) #4 kudu::MemoryTrackingBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.cc:308:22 (libkudu_util.so+0x0000001d39bf) #5 kudu::BufferAllocator::BestEffortAllocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/memory.h:131:22 (libkudu_util.so+0x0000001cef0b) #6 kudu::ArenaBase<true>::NewComponent(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/arena.cc:113:20 (libkudu_util.so+0x0000001cdd34) #7 kudu::ArenaBase<true>::AllocateBytesFallback(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/arena.cc:93:26 (libkudu_util.so+0x0000001ce585) #8 kudu::ArenaBase<true>::AllocateBytesAligned(unsigned long, unsigned long) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/memory/arena.h:442:10 (libtablet.so+0x0000002a11e9) #9 kudu::tablet::Mutation* kudu::tablet::Mutation::CreateInArena<kudu::ThreadSafeMemoryTrackingArena>(kudu::ThreadSafeMemoryTrackingArena*, kudu::Timestamp, kudu::RowChangeList const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mutation.h:115:19 (libtablet.so+0x000000298934) #10 kudu::tablet::MemRowSet::Reinsert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::tablet::MRSRow*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/memrowset.cc:195:19 (libtablet.so+0x0000002953cb) #11 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/memrowset.cc:156:14 (libtablet.so+0x000000294f39) #12 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001e5f6d) #13 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001e6f0d) #14 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001e6d5f) #15 kudu::tablet::LocalTabletWriter::WriteBatch(std::vector<kudu::tablet::LocalTabletWriter::Op, std::allocator<kudu::tablet::LocalTabletWriter::Op> > const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:103:5 (mt-tablet-test+0x0000004f9bea) #16 kudu::tablet::LocalTabletWriter::Write(kudu::RowOperationsPB_Type, kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:84:12 (mt-tablet-test+0x0000004f98bd) #17 kudu::tablet::LocalTabletWriter::Insert(kudu::KuduPartialRow const&) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/local_tablet_writer.h:63:12 (mt-tablet-test+0x0000004f928e) #18 kudu::tablet::TabletTestBase<kudu::tablet::StringKeyTestSetup>::InsertTestRow(kudu::tablet::LocalTabletWriter*, long, int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet-test-base.h:366:12 (mt-tablet-test+0x00000051dd71) #19 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::DeleteAndReinsertCycleThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:330:9 (mt-tablet-test+0x00000051d84f) #20 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed) #21 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a) #22 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3) #23 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1) #24 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561) #25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af) Mutex M15465 (0x7d500001c958) created at: #0 pthread_mutex_trylock /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1194 (mt-tablet-test+0x000000464f53) #1 __gthread_mutex_trylock(pthread_mutex_t*) /home/jenkins-slave/workspace/kudu-1/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/bits/gthr-default.h:757:12 (libtablet.so+0x0000001efee6) #2 std::mutex::try_lock() /home/jenkins-slave/workspace/kudu-1/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/mutex:146:15 (libtablet.so+0x000000205d59) #3 std::unique_lock<std::mutex>::unique_lock(std::mutex&, std::try_to_lock_t) /home/jenkins-slave/workspace/kudu-1/thirdparty/installed-deps-tsan/gcc/include/c++/4.9.3/mutex:415:34 (libtablet.so+0x0000001f4dcf) #4 kudu::tablet::Tablet::ReplaceMemRowSetUnlocked(kudu::tablet::RowSetsInCompaction*, std::shared_ptr<kudu::tablet::MemRowSet>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:720:32 (libtablet.so+0x0000001e7af7) #5 kudu::tablet::Tablet::FlushUnlocked() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:704:5 (libtablet.so+0x0000001e7972) #6 kudu::tablet::Tablet::Flush() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/tablet.cc:694:10 (libtablet.so+0x0000001e77e0) #7 kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::FlushThread(int) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:269:9 (mt-tablet-test+0x0000004f461d) #8 boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>::operator()(kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int) const /usr/include/boost/bind/mem_fn_template.hpp:165:29 (mt-tablet-test+0x0000004f7aed) #9 void boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> >::operator()<boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:313:9 (mt-tablet-test+0x0000004f7a4a) #10 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:27 (mt-tablet-test+0x0000004f79b3) #11 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>, int>, boost::_bi::list2<boost::_bi::value<kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*>, boost::_bi::value<int> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (mt-tablet-test+0x0000004f77b1) #12 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:766:14 (libkrpc.so+0x0000000cd561) #13 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021e5af) Thread T14 'test4-8671' (tid=8671, running) created by main thread at: #0 pthread_create /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (mt-tablet-test+0x000000486403) #1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021de37) #2 kudu::Status kudu::Thread::Create<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int>(std::string const&, std::string const&, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>* const&, int const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:162:12 (mt-tablet-test+0x0000004f727d) #3 void kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::StartThreads<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int)>(int, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int)) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:376:7 (mt-tablet-test+0x0000004f3d9a) #4 kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::StringKeyTestSetup>::TestBody() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:451:3 (mt-tablet-test+0x00000051d642) #5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10 (libgmock.so+0x0000000442f9) #6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114 (libgmock.so+0x0000000442f9) #7 main /home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:48:13 (libkudu_test_main.so+0x000000001e75) Thread T4 'test0-8661' (tid=8661, running) created by main thread at: #0 pthread_create /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (mt-tablet-test+0x000000486403) #1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021de37) #2 kudu::Status kudu::Thread::Create<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int>(std::string const&, std::string const&, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>* const&, int const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:162:12 (mt-tablet-test+0x0000004f727d) #3 void kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::StartThreads<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int)>(int, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int)) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:376:7 (mt-tablet-test+0x0000004f3d9a) #4 kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::StringKeyTestSetup>::TestBody() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:444:3 (mt-tablet-test+0x00000051d558) #5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10 (libgmock.so+0x0000000442f9) #6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114 (libgmock.so+0x0000000442f9) #7 main /home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:48:13 (libkudu_test_main.so+0x000000001e75) Thread T19 'test9-8677' (tid=8677, running) created by main thread at: #0 pthread_create /home/jenkins-slave/workspace/kudu-1/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (mt-tablet-test+0x000000486403) #1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021de37) #2 kudu::Status kudu::Thread::Create<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>*, int>(std::string const&, std::string const&, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int), kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>* const&, int const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-1/src/kudu/util/thread.h:162:12 (mt-tablet-test+0x0000004f727d) #3 void kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::StartThreads<void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::*)(int)>(int, void (kudu::tablet::MultiThreadedTabletTest<kudu::tablet::StringKeyTestSetup>::* const&)(int)) /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:376:7 (mt-tablet-test+0x0000004f3d9a) #4 kudu::tablet::MultiThreadedTabletTest_DeleteAndReinsert_Test<kudu::tablet::StringKeyTestSetup>::TestBody() /home/jenkins-slave/workspace/kudu-1/src/kudu/tablet/mt-tablet-test.cc:451:3 (mt-tablet-test+0x00000051d642) #5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2078:10 (libgmock.so+0x0000000442f9) #6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-1/thirdparty/gmock-1.7.0/gtest/src/gtest.cc:2114 (libgmock.so+0x0000000442f9) #7 main /home/jenkins-slave/workspace/kudu-1/src/kudu/util/test_main.cc:48:13 (libkudu_test_main.so+0x000000001e75)