Uploaded image for project: 'Kudu'
  1. Kudu
  2. KUDU-2916

Admin.TestDumpMemTrackers is flaky in tsan

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Major
    • Resolution: Duplicate
    • Affects Version/s: 1.10.0
    • Fix Version/s: n/a
    • Component/s: None
    • Labels:
      None

      Description

      I saw a tsan failure for AdminCliTest.TestDumpMemTrackers with the following log:

      /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-admin-test.cc:2162: Failure
      Value of: s.ok()
        Actual: false
      Expected: true
      Runtime error: /tmp/dist-test-taskUWtx7r/build/tsan/bin/kudu: process exited with non-zero status 66
      stdout: {"id":"root","limit":-1,"current_consumption":481,"peak_consumption":481,"child_trackers":[{"id":"server","parent_id":"root","limit":-1,"current_consumption":313,"peak_consumption":313,"child_trackers":[{"id":"result-tracker","parent_id":"server","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_block_manager","parent_id":"server","limit":-1,"current_consumption":48,"peak_consumption":48},{"id":"tablet-9276b163452b4b0399ff2cae579f7251","parent_id":"server","limit":-1,"current_consumption":265,"peak_consumption":265,"child_trackers":[{"id":"DeltaMemStores","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"MemRowSet-0","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":265,"peak_consumption":265},{"id":"txn_tracker","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":67108864,"current_consumption":0,"peak_consumption":0}]}]},{"id":"ttl-cache-sharded_fifo_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"code_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"block_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"lbm-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_cache","parent_id":"root","limit":1073741824,"current_consumption":168,"peak_consumption":168,"child_trackers":[{"id":"log_cache:457a3168758d4f4f8f4c59e8dd179cd3:9276b163452b4b0399ff2cae579f7251","parent_id":"log_cache","limit":10485760,"current_consumption":168,"peak_consumption":168}]}]}
      
      stderr: W0803 14:03:00.206982  8443 flags.cc:404] Enabled unsafe flag: --never_fsync=true
      W0803 14:03:00.849385  8443 thread.cc:599] rpc reactor (reactor) Time spent creating pthread: real 0.590s	user 0.230s	sys 0.360s
      W0803 14:03:00.849658  8443 thread.cc:566] rpc reactor (reactor) Time spent starting thread: real 0.591s	user 0.230s	sys 0.360s
      ==================
      WARNING: ThreadSanitizer: destroy of a locked mutex (pid=8443)
          #0 pthread_rwlock_destroy /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1313 (kudu+0x4bbb24)
          #1 glog_internal_namespace_::Mutex::~Mutex() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30 (libglog.so.0+0x16488)
          #2 cxa_at_exit_wrapper(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:386 (kudu+0x48beb3)
      
        and:
          #0 operator new(unsigned long) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:57 (kudu+0x52ae83)
          #1 __allocate /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/new:228:10 (libc++.so.1+0xd63f3)
          #2 allocate /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1793 (libc++.so.1+0xd63f3)
          #3 allocate /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1547 (libc++.so.1+0xd63f3)
          #4 __init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1591 (libc++.so.1+0xd63f3)
          #5 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::basic_string(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1653 (libc++.so.1+0xd63f3)
          #6 std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >::pair(std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/utility:324:5 (libprotobuf.so.14+0x188711)
          #7 void std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> >::construct<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1805 (libprotobuf.so.14+0x188711)
          #8 void std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> > >::__construct<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::integral_constant<bool, true>, std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> >&, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1715 (libprotobuf.so.14+0x188711)
          #9 void std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> > >::construct<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> >&, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1561 (libprotobuf.so.14+0x188711)
          #10 std::__1::unique_ptr<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*>, std::__1::__tree_node_destructor<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> > > > std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, true>, std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> > > >::__construct_node<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2192 (libprotobuf.so.14+0x188711)
          #11 std::__1::pair<std::__1::__tree_iterator<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*>*, long>, bool> std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, true>, std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> > > >::__emplace_unique_key_args<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2137 (libprotobuf.so.14+0x188711)
          #12 bool google::protobuf::InsertIfNotPresent<std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > > >(std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > >*, std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > >::value_type::first_type const&, std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > >::value_type::second_type const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:1273:16 (libprotobuf.so.14+0x184c18)
          #13 google::protobuf::SimpleDescriptorDatabase::DescriptorIndex<std::__1::pair<void const*, int> >::AddFile(google::protobuf::FileDescriptorProto const&, std::__1::pair<void const*, int>) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:57 (libprotobuf.so.14+0x184c18)
          #14 google::protobuf::EncodedDescriptorDatabase::Add(void const*, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:316:19 (libprotobuf.so.14+0x182d7c)
          #15 google::protobuf::DescriptorPool::InternalAddGeneratedFile(void const*, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor.cc:1315:3 (libprotobuf.so.14+0x13b705)
          #16 google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::(anonymous namespace)::AddDescriptorsImpl() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:240:3 (libprotobuf.so.14+0x237c10)
          #17 google::protobuf::internal::FunctionClosure0::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5 (libprotobuf.so.14+0xd330b)
          #18 google::protobuf::GoogleOnceInitImpl(long*, google::protobuf::Closure*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14 (libprotobuf.so.14+0xd5d6a)
          #19 google::protobuf::GoogleOnceInit(long*, void (*)()) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5 (libprotobuf.so.14+0x237b66)
          #20 google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:251 (libprotobuf.so.14+0x237b66)
          #21 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::(anonymous namespace)::AddDescriptorsImpl() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:179:3 (libprotobuf.so.14+0x12ca1d)
          #22 google::protobuf::internal::FunctionClosure0::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5 (libprotobuf.so.14+0xd330b)
          #23 google::protobuf::GoogleOnceInitImpl(long*, google::protobuf::Closure*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14 (libprotobuf.so.14+0xd5d6a)
          #24 google::protobuf::GoogleOnceInit(long*, void (*)()) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5 (libprotobuf.so.14+0xcf0a6)
          #25 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::AddDescriptors() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:185 (libprotobuf.so.14+0xcf0a6)
          #26 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::StaticDescriptorInitializer::StaticDescriptorInitializer() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:190 (libprotobuf.so.14+0xcf0a6)
          #27 __cxx_global_var_init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:192 (libprotobuf.so.14+0xcf0a6)
          #28 _GLOBAL__sub_I_api.pb.cc /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc (libprotobuf.so.14+0xcf0a6)
          #29 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78 (ld-linux-x86-64.so.2+0x102d9)
      
        Location is global 'google::vmodule_lock' of size 64 at 0x7f67b632f878 (libglog.so.0+0x00000023c878)
      
        Mutex M134 (0x7f67b632f878) created at:
          #0 pthread_rwlock_init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1304 (kudu+0x4bbd94)
          #1 glog_internal_namespace_::Mutex::Mutex() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:247:19 (libglog.so.0+0xcca6)
          #2 __cxx_global_var_init.15 /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:120 (libglog.so.0+0xcca6)
          #3 _GLOBAL__sub_I_vlog_is_on.cc /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc (libglog.so.0+0xcca6)
          #4 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78 (ld-linux-x86-64.so.2+0x102d9)
      
      SUMMARY: ThreadSanitizer: destroy of a locked mutex /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30 in glog_internal_namespace_::Mutex::~Mutex()
      ==================
      ==================
      WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=8443)
        Cycle in lock order graph: M1860 (0x7b14000167b8) => M37857730294380664 (0x000000000000) => M1860
      
        Mutex M37857730294380664 acquired here while holding mutex M1860 in main thread:
          #0 pthread_rwlock_wrlock /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1352 (kudu+0x4bb154)
          #1 glog_internal_namespace_::Mutex::Lock() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30 (libglog.so.0+0x1abb7)
          #2 glog_internal_namespace_::MutexLock::MutexLock(glog_internal_namespace_::Mutex*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:290 (libglog.so.0+0x1abb7)
          #3 google::InitVLOG3__(int**, int*, char const*, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:199 (libglog.so.0+0x1abb7)
          #4 kudu::rpc::Messenger::ShutdownInternal(kudu::rpc::Messenger::ShutdownMode) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:283:5 (libkrpc.so+0xa9471)
          #5 kudu::rpc::Messenger::AllExternalReferencesDropped() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:249:3 (libkrpc.so+0xa9227)
          #6 std::__1::mem_fun_t<void, kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1120:17 (libkrpc.so+0xad755)
          #7 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*, std::__1::mem_fun_t<void, kudu::rpc::Messenger>, std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586 (libkrpc.so+0xad755)
          #8 std::__1::__shared_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9 (kudu+0x576f2e)
          #9 std::__1::__shared_weak_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532 (kudu+0x576f2e)
          #10 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468 (kudu+0x576f2e)
          #11 kudu::rpc::Proxy::~Proxy() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/proxy.cc:72:1 (libkrpc.so+0xbd0f3)
          #12 kudu::server::GenericServiceProxy::~GenericServiceProxy() /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/server/server_base.proxy.cc:26:1 (libserver_base_proto.so+0x4bd79)
          #13 std::__1::default_delete<kudu::server::GenericServiceProxy>::operator()(kudu::server::GenericServiceProxy*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5 (libkudu_tools_util.so+0x74acf)
          #14 std::__1::unique_ptr<kudu::server::GenericServiceProxy, std::__1::default_delete<kudu::server::GenericServiceProxy> >::reset(kudu::server::GenericServiceProxy*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2598 (libkudu_tools_util.so+0x74acf)
          #15 std::__1::unique_ptr<kudu::server::GenericServiceProxy, std::__1::default_delete<kudu::server::GenericServiceProxy> >::~unique_ptr() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2552 (libkudu_tools_util.so+0x74acf)
          #16 kudu::tools::DumpMemTrackers(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned short) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:629 (libkudu_tools_util.so+0x74acf)
          #17 kudu::tools::(anonymous namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10 (kudu+0x5b8599)
          #18 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_ /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 (kudu+0x53956b)
          #19 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318 (kudu+0x53956b)
          #20 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 (kudu+0x53949d)
          #21 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 (libkudu_tools_util.so+0x6b2a4)
          #22 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10 (libkudu_tools_util.so+0x69974)
          #23 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15 (kudu+0x5bbcf6)
          #24 kudu::tools::RunTool(int, char**, bool) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16 (kudu+0x5bcc51)
          #25 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10 (kudu+0x5bcfbe)
      
          Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative warning message
      
        Mutex M1860 acquired here while holding mutex M37857730294380664 in thread T7:
          #0 AnnotateRWLockAcquired /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc:271 (kudu+0x4dbd0c)
          #1 kudu::rw_spinlock::lock() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:112:5 (libkudu_client.so+0x16d842)
          #2 kudu::percpu_rwlock::lock() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:222:22 (libkudu_client.so+0x16d7d2)
          #3 std::__1::lock_guard<kudu::percpu_rwlock>::lock_guard(kudu::percpu_rwlock&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__mutex_base:104:27 (libkrpc.so+0xaad39)
          #4 kudu::rpc::Messenger::~Messenger() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:430 (libkrpc.so+0xaad39)
          #5 std::__1::default_delete<kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5 (libkrpc.so+0xb081b)
          #6 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*, std::__1::default_delete<kudu::rpc::Messenger>, std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586 (libkrpc.so+0xb081b)
          #7 std::__1::__shared_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9 (kudu+0x576f2e)
          #8 std::__1::__shared_weak_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532 (kudu+0x576f2e)
          #9 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468 (kudu+0x576f2e)
          #10 std::__1::shared_ptr<kudu::rpc::Messenger>::reset() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4603:5 (libkrpc.so+0xbf151)
          #11 kudu::rpc::ReactorThread::RunThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:499 (libkrpc.so+0xbf151)
          #12 boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xc9539)
          #13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xc948a)
          #14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xc9413)
          #15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xc9209)
          #16 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb8a11)
          #17 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:657:3 (libkudu_util.so+0x1e1824)
      
        Thread T7 'rpc reactor-845' (tid=8451, running) created by main thread at:
          #0 pthread_create /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992 (kudu+0x4bd8eb)
          #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:601:15 (libkudu_util.so+0x1e100b)
          #2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12 (libkrpc.so+0xc48e5)
          #3 kudu::rpc::ReactorThread::Init() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:185:10 (libkrpc.so+0xbec4e)
          #4 kudu::rpc::Reactor::Init() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:759:18 (libkrpc.so+0xc32f1)
          #5 kudu::rpc::Messenger::Init() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:446:5 (libkrpc.so+0xa90e2)
          #6 kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:205:3 (libkrpc.so+0xa8b3d)
          #7 kudu::Status kudu::tools::BuildProxy<kudu::server::GenericServiceProxy>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned short, std::__1::unique_ptr<kudu::server::GenericServiceProxy, std::__1::default_delete<kudu::server::GenericServiceProxy> >*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:328:3 (libkudu_tools_util.so+0x7c55e)
          #8 kudu::tools::DumpMemTrackers(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned short) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:591:3 (libkudu_tools_util.so+0x747e3)
          #9 kudu::tools::(anonymous namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10 (kudu+0x5b8599)
          #10 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_ /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 (kudu+0x53956b)
          #11 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318 (kudu+0x53956b)
          #12 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 (kudu+0x53949d)
          #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 (libkudu_tools_util.so+0x6b2a4)
          #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10 (libkudu_tools_util.so+0x69974)
          #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15 (kudu+0x5bbcf6)
          #16 kudu::tools::RunTool(int, char**, bool) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16 (kudu+0x5bcc51)
          #17 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10 (kudu+0x5bcfbe)
      
      SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30 in glog_internal_namespace_::Mutex::Lock()
      ==================
      ThreadSanitizer: reported 2 warnings
      

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                Unassigned
                Reporter:
                hahao Hao Hao
              • Votes:
                0 Vote for this issue
                Watchers:
                1 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: