Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Fixed
-
Impala 4.0.0
-
None
-
ghx-label-2
Description
TSAN is reporting a data race in AdmissionController::WaitOnQueued:
WARNING: ThreadSanitizer: data race (pid=4257)
Write of size 8 at 0x7b58000901b0 by thread T416:
#0 std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_erase(unsigned long, std::__detail::_Hash_node_base*, std::__detail::_Hash_node<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, true>*) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:1891:7 (impalad+0x22bbe78)
#1 std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_erase(std::integral_constant<bool, true>, impala::UniqueIdPB const&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:1916:7 (impalad+0x22bbd3a)
#2 std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::erase(impala::UniqueIdPB const&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:759:16 (impalad+0x22bbca0)
#3 std::unordered_map<impala::UniqueIdPB, impala::AdmissionController::QueueNode, std::hash<impala::UniqueIdPB>, std::equal_to<impala::UniqueIdPB>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> > >::erase(impala::UniqueIdPB const&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/unordered_map.h:814:21 (impalad+0x22bbc50)
#4 impala::AdmissionController::WaitOnQueued(impala::UniqueIdPB const&, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*, long, bool*)::$_6::operator()() const /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/scheduling/admission-controller.cc:1278:49 (impalad+0x229d199)
#5 impala::ScopeExitTrigger<impala::AdmissionController::WaitOnQueued(impala::UniqueIdPB const&, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*, long, bool*)::$_6>::~ScopeExitTrigger() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/scope-exit-trigger.h:40:25 (impalad+0x2294ca9)
#6 impala::AdmissionController::WaitOnQueued(impala::UniqueIdPB const&, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*, long, bool*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/scheduling/admission-controller.cc:1333:1 (impalad+0x2294912)
#7 impala::LocalAdmissionControlClient::SubmitForAdmission(impala::AdmissionController::AdmissionRequest const&, impala::RuntimeProfile::EventSequence*, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/scheduling/local-admission-control-client.cc:45:62 (impalad+0x2c1e40e)
#8 impala::ClientRequestState::FinishExecQueryOrDmlRequest() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/client-request-state.cc:578:52 (impalad+0x245c651)
#9 boost::_mfi::mf0<void, impala::ClientRequestState>::operator()(impala::ClientRequestState*) const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:49:29 (impalad+0x2468ef6)
#10 void boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> >::operator()<boost::_mfi::mf0<void, impala::ClientRequestState>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, impala::ClientRequestState>&, boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:259:9 (impalad+0x2468e4a)
#11 boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ClientRequestState>, boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x2468dd3)
#12 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ClientRequestState>, boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 (impalad+0x2468bc9)
#13 boost::function0<void>::operator()() const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 (impalad+0x203c831)
#14 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.cc:360:3 (impalad+0x265502b)
#15 void boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 (impalad+0x265ce4c)
#16 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x265cd63)
#17 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 (impalad+0x265ca50)
#18 thread_proxy <null> (impalad+0x33ab591)
Previous write of size 8 at 0x7b58000901b0 by thread T540:
#0 std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_erase(unsigned long, std::__detail::_Hash_node_base*, std::__detail::_Hash_node<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, true>*) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:1891:7 (impalad+0x22bbe78)
#1 std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_erase(std::integral_constant<bool, true>, impala::UniqueIdPB const&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:1916:7 (impalad+0x22bbd3a)
#2 std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::erase(impala::UniqueIdPB const&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:759:16 (impalad+0x22bbca0)
#3 std::unordered_map<impala::UniqueIdPB, impala::AdmissionController::QueueNode, std::hash<impala::UniqueIdPB>, std::equal_to<impala::UniqueIdPB>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> > >::erase(impala::UniqueIdPB const&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/unordered_map.h:814:21 (impalad+0x22bbc50)
#4 impala::AdmissionController::WaitOnQueued(impala::UniqueIdPB const&, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*, long, bool*)::$_6::operator()() const /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/scheduling/admission-controller.cc:1278:49 (impalad+0x229d199)
#5 impala::ScopeExitTrigger<impala::AdmissionController::WaitOnQueued(impala::UniqueIdPB const&, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*, long, bool*)::$_6>::~ScopeExitTrigger() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/scope-exit-trigger.h:40:25 (impalad+0x2294ca9)
#6 impala::AdmissionController::WaitOnQueued(impala::UniqueIdPB const&, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*, long, bool*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/scheduling/admission-controller.cc:1333:1 (impalad+0x2294912)
#7 impala::LocalAdmissionControlClient::SubmitForAdmission(impala::AdmissionController::AdmissionRequest const&, impala::RuntimeProfile::EventSequence*, std::unique_ptr<impala::QuerySchedulePB, std::default_delete<impala::QuerySchedulePB> >*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/scheduling/local-admission-control-client.cc:45:62 (impalad+0x2c1e40e)
#8 impala::ClientRequestState::FinishExecQueryOrDmlRequest() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/client-request-state.cc:578:52 (impalad+0x245c651)
#9 boost::_mfi::mf0<void, impala::ClientRequestState>::operator()(impala::ClientRequestState*) const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:49:29 (impalad+0x2468ef6)
#10 void boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> >::operator()<boost::_mfi::mf0<void, impala::ClientRequestState>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, impala::ClientRequestState>&, boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:259:9 (impalad+0x2468e4a)
#11 boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ClientRequestState>, boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x2468dd3)
#12 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ClientRequestState>, boost::_bi::list1<boost::_bi::value<impala::ClientRequestState*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 (impalad+0x2468bc9)
#13 boost::function0<void>::operator()() const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 (impalad+0x203c831)
#14 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.cc:360:3 (impalad+0x265502b)
#15 void boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 (impalad+0x265ce4c)
#16 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x265cd63)
#17 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 (impalad+0x265ca50)
#18 thread_proxy <null> (impalad+0x33ab591)
Location is heap block of size 728 at 0x7b5800090000 allocated by main thread:
#0 operator new(unsigned long) /mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:44 (impalad+0x1b36126)
#1 impala::ExecEnv::ExecEnv(int, int, int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/runtime/exec-env.cc:301:7 (impalad+0x2122a47)
#2 impala::ExecEnv::ExecEnv() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/runtime/exec-env.cc:239:5 (impalad+0x2122019)
#3 ImpaladMain(int, char**) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impalad-main.cc:70:11 (impalad+0x23e0545)
#4 main /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/daemon-main.cc:37:12 (impalad+0x1b389ca)
Thread T416 (tid=20718, running) created by thread T586 at:
#0 pthread_create /mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889 (impalad+0x1aa9c76)
#1 boost::thread::start_thread_noexcept() <null> (impalad+0x33aa5e8)
#2 boost::thread::thread<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, (impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, (impala::PromiseMode)0>*) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13 (impalad+0x2657814)
#3 impala::Thread::StartThread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.cc:317:13 (impalad+0x26549a1)
#4 impala::Status impala::Thread::Create<void (impala::ClientRequestState::*)(), impala::ClientRequestState*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, void (impala::ClientRequestState::* const&)(), impala::ClientRequestState* const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.h:81:12 (impalad+0x2464477)
#5 impala::ClientRequestState::ExecAsyncQueryOrDmlRequest(impala::TQueryExecRequest const&) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/client-request-state.cc:569:3 (impalad+0x2458747)
#6 impala::ClientRequestState::Exec() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/client-request-state.cc:231:7 (impalad+0x2457119)
#7 impala::ImpalaServer::ExecuteInternal(impala::TQueryCtx const&, std::shared_ptr<impala::ImpalaServer::SessionState>, bool*, impala::QueryHandle*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impala-server.cc:1161:3 (impalad+0x23eded0)
#8 impala::ImpalaServer::Execute(impala::TQueryCtx*, std::shared_ptr<impala::ImpalaServer::SessionState>, impala::QueryHandle*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impala-server.cc:1094:19 (impalad+0x23ed8bb)
#9 impala::ImpalaServer::ExecuteStatement(apache::hive::service::cli::thrift::TExecuteStatementResp&, apache::hive::service::cli::thrift::TExecuteStatementReq const&) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impala-hs2-server.cc:499:12 (impalad+0x248218b)
#10 impala::ChildQuery::ExecAndFetch() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/child-query.cc:63:19 (impalad+0x24b08f2)
#11 impala::ChildQueryExecutor::ExecChildQueries() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/child-query.cc:219:33 (impalad+0x24b7882)
#12 boost::_mfi::mf0<void, impala::ChildQueryExecutor>::operator()(impala::ChildQueryExecutor*) const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:49:29 (impalad+0x24b99c6)
#13 void boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> >::operator()<boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, impala::ChildQueryExecutor>&, boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:259:9 (impalad+0x24b991a)
#14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x24b98a3)
#15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 (impalad+0x24b9699)
#16 boost::function0<void>::operator()() const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 (impalad+0x203c831)
#17 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.cc:360:3 (impalad+0x265502b)
#18 void boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 (impalad+0x265ce4c)
#19 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x265cd63)
#20 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 (impalad+0x265ca50)
#21 thread_proxy <null> (impalad+0x33ab591)
Thread T540 (tid=20731, running) created by thread T561 at:
#0 pthread_create /mnt/source/llvm/llvm-5.0.1.src-p3/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:889 (impalad+0x1aa9c76)
#1 boost::thread::start_thread_noexcept() <null> (impalad+0x33aa5e8)
#2 boost::thread::thread<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, (impala::PromiseMode)0>*>(void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::function<void ()>, impala::ThreadDebugInfo*, impala::Promise<long, (impala::PromiseMode)0>*) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:420:13 (impalad+0x2657814)
#3 impala::Thread::StartThread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()> const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.cc:317:13 (impalad+0x26549a1)
#4 impala::Status impala::Thread::Create<void (impala::ClientRequestState::*)(), impala::ClientRequestState*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, void (impala::ClientRequestState::* const&)(), impala::ClientRequestState* const&, std::unique_ptr<impala::Thread, std::default_delete<impala::Thread> >*, bool) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.h:81:12 (impalad+0x2464477)
#5 impala::ClientRequestState::ExecAsyncQueryOrDmlRequest(impala::TQueryExecRequest const&) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/client-request-state.cc:569:3 (impalad+0x2458747)
#6 impala::ClientRequestState::Exec() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/client-request-state.cc:231:7 (impalad+0x2457119)
#7 impala::ImpalaServer::ExecuteInternal(impala::TQueryCtx const&, std::shared_ptr<impala::ImpalaServer::SessionState>, bool*, impala::QueryHandle*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impala-server.cc:1161:3 (impalad+0x23eded0)
#8 impala::ImpalaServer::Execute(impala::TQueryCtx*, std::shared_ptr<impala::ImpalaServer::SessionState>, impala::QueryHandle*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impala-server.cc:1094:19 (impalad+0x23ed8bb)
#9 impala::ImpalaServer::ExecuteStatement(apache::hive::service::cli::thrift::TExecuteStatementResp&, apache::hive::service::cli::thrift::TExecuteStatementReq const&) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/impala-hs2-server.cc:499:12 (impalad+0x248218b)
#10 impala::ChildQuery::ExecAndFetch() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/child-query.cc:63:19 (impalad+0x24b08f2)
#11 impala::ChildQueryExecutor::ExecChildQueries() /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/service/child-query.cc:219:33 (impalad+0x24b7882)
#12 boost::_mfi::mf0<void, impala::ChildQueryExecutor>::operator()(impala::ChildQueryExecutor*) const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/mem_fn_template.hpp:49:29 (impalad+0x24b99c6)
#13 void boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> >::operator()<boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, impala::ChildQueryExecutor>&, boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:259:9 (impalad+0x24b991a)
#14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x24b98a3)
#15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, impala::ChildQueryExecutor>, boost::_bi::list1<boost::_bi::value<impala::ChildQueryExecutor*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:159:11 (impalad+0x24b9699)
#16 boost::function0<void>::operator()() const /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/function/function_template.hpp:770:14 (impalad+0x203c831)
#17 impala::Thread::SuperviseThread(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*) /data/jenkins/workspace/impala-asf-master-core-tsan/repos/Impala/be/src/util/thread.cc:360:3 (impalad+0x265502b)
#18 void boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> >::operator()<void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list0&, int) /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:531:9 (impalad+0x265ce4c)
#19 boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > >::operator()() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/bind/bind.hpp:1222:16 (impalad+0x265cd63)
#20 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()>, impala::ThreadDebugInfo const*, impala::Promise<long, (impala::PromiseMode)0>*), boost::_bi::list5<boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::ThreadDebugInfo*>, boost::_bi::value<impala::Promise<long, (impala::PromiseMode)0>*> > > >::run() /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/boost-1.61.0-p2/include/boost/thread/detail/thread.hpp:116:17 (impalad+0x265ca50)
#21 thread_proxy <null> (impalad+0x33ab591)
SUMMARY: ThreadSanitizer: data race /data/jenkins/workspace/impala-asf-master-core-tsan/Impala-Toolchain/toolchain-packages-gcc7.5.0/gcc-7.5.0/lib/gcc/x86_64-pc-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/hashtable.h:1891:7 in std::_Hashtable<impala::UniqueIdPB, std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, std::allocator<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode> >, std::__detail::_Select1st, std::equal_to<impala::UniqueIdPB>, std::hash<impala::UniqueIdPB>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_erase(unsigned long, std::__detail::_Hash_node_base*, std::__detail::_Hash_node<std::pair<impala::UniqueIdPB const, impala::AdmissionController::QueueNode>, true>*)
==================
twmarshall I'm assigning this to you as WaitOnQueued was introduced in IMPALA-9930 (part 2). Feel free to reassign if you think the issue is not related to your change.