Details
-
Bug
-
Status: Resolved
-
Critical
-
Resolution: Fixed
-
Impala 2.9.0, Impala 2.10.0
-
ghx-label-3
Description
There seems to be a race when queries are being cancelled and the client tries to fetch results at the same time. I used the following loop to provoke a use-after-free detected by asan:
while [ $? -eq 0 ]; do impala-py.test tests/query_test/test_scanners.py::TestParquet::test_corrupt_files --exploration_strategy=exhaustive -n8; done
To run this I had to make the following change to {{query_test/test_scanners.py
- def test_corrupt_files(self, vector):
+ @pytest.mark.parametrize('multiplier', xrange(32))
+ def test_corrupt_files(self, vector, multiplier):
This it the logging produced by asan:
Log file created at: 2017/06/22 11:15:36
Running on machine: lv-desktop
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
E0622 11:15:36.630154 11764 logging.cc:124] stderr will be logged to this file.
=================================================================
==11764==ERROR: AddressSanitizer: heap-use-after-free on address 0x6120003d9bf8 at pc 0x7ff0abd21b24 bp 0x7fee287f49e0 sp 0x7fee287f49d8
READ of size 8 at 0x6120003d9bf8 thread T193
#0 0x7ff0abd21b23 in __gnu_cxx::__normal_iterator<impala::MemTracker**, std::vector<impala::MemTracker*, std::allocator<impala::MemTracker*> > >::__normal_iterator(impala::MemTracker** const&) /opt/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/stl_iterator.h:729:20
#1 0x7ff0abd21a3b in std::vector<impala::MemTracker*, std::allocator<impala::MemTracker*> >::end() /opt/Impala-Toolchain/gcc-4.9.2/lib/gcc/x86_64-unknown-linux-gnu/4.9.2/../../../../include/c++/4.9.2/bits/stl_vector.h:566:16
#2 0x7ff0ab202faa in impala::MemTracker::AnyLimitExceeded() /home/lv/i3/be/src/runtime/mem-tracker.h:224:21
#3 0x7ff0ab2bb394 in impala::RuntimeState::CheckQueryState() /home/lv/i3/be/src/runtime/runtime-state.cc:244:30
#4 0x7ff0aa9c0de4 in impala::PlanRootSink::GetNext(impala::RuntimeState*, impala::QueryResultSet*, int, bool*) /home/lv/i3/be/src/exec/plan-root-sink.cc:163:10
#5 0x7ff0ab0eecda in impala::Coordinator::GetNext(impala::QueryResultSet*, int, bool*) /home/lv/i3/be/src/runtime/coordinator.cc:876:19
#6 0x7ff0a94c8b34 in impala::ClientRequestState::FetchRowsInternal(int, impala::QueryResultSet*) /home/lv/i3/be/src/service/client-request-state.cc:787:21
#7 0x7ff0a94c81f4 in impala::ClientRequestState::FetchRows(int, impala::QueryResultSet*) /home/lv/i3/be/src/service/client-request-state.cc:692:28
#8 0x7ff0a94ae4c9 in impala::ImpalaServer::FetchInternal(impala::TUniqueId const&, bool, int, beeswax::Results*) /home/lv/i3/be/src/service/impala-beeswax-server.cc:525:25
#9 0x7ff0a94ad9d6 in impala::ImpalaServer::fetch(beeswax::Results&, beeswax::QueryHandle const&, bool, int) /home/lv/i3/be/src/service/impala-beeswax-server.cc:171:19
#10 0x7ff0a89127c6 in beeswax::BeeswaxServiceProcessor::process_fetch(int, apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, void*) /home/lv/i3/be/generated-sources/gen-cpp/BeeswaxService.cpp:3150:5
#11 0x7ff0a890fff9 in beeswax::BeeswaxServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, std::string const&, int, void*) /home/lv/i3/be/generated-sources/gen-cpp/BeeswaxService.cpp:2952:3
#12 0x7ff0a88dceca in impala::ImpalaServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, std::string const&, int, void*) /home/lv/i3/be/generated-sources/gen-cpp/ImpalaService.cpp:1673:12
#13 0x7ff0abdd873a in apache::thrift::TDispatchProcessor::process(boost::shared_ptr<apache::thrift::protocol::TProtocol>, boost::shared_ptr<apache::thrift::protocol::TProtocol>, void*) /opt/Impala-Toolchain/thrift-0.9.0-p8/include/thrift/TDispatchProcessor.h:121:12
#14 0x16e6f3a in apache::thrift::server::TThreadPoolServer::Task::run() (/home/lv/i3/be/build/debug/service/impalad+0x16e6f3a)
#15 0x16ca448 in apache::thrift::concurrency::ThreadManager::Worker::run() (/home/lv/i3/be/build/debug/service/impalad+0x16ca448)
#16 0x7ff0a97cd239 in impala::ThriftThread::RunRunnable(boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*) /home/lv/i3/be/src/rpc/thrift-thread.cc:64:3
#17 0x7ff0a97cf87c in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator()(impala::ThriftThread*, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*) const /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/mem_fn_template.hpp:280:16
#18 0x7ff0a97cf6c3 in void boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>&, boost::_bi::list0&, int) /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind.hpp:392:9
#19 0x7ff0a97cf567 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator()() /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind_template.hpp:20:16
#20 0x7ff0ab8aa6d2 in boost::function0<void>::operator()() const /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/function/function_template.hpp:766:14
#21 0x7ff0ab8a718d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /home/lv/i3/be/src/util/thread.cc:322:3
#22 0x7ff0ab8b1d8a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind.hpp:457:9
#23 0x7ff0ab8b1c17 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind_template.hpp:20:16
#24 0x94ff09 in thread_proxy (/home/lv/i3/be/build/debug/service/impalad+0x94ff09)
#25 0x7ff0a5182183 in start_thread /build/eglibc-MjiXCM/eglibc-2.19/nptl/pthread_create.c:312
#26 0x7ff0a4c99bec in clone /build/eglibc-MjiXCM/eglibc-2.19/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111
0x6120003d9bf8 is located 184 bytes inside of 296-byte region [0x6120003d9b40,0x6120003d9c68)
freed by thread T98430 here:
#0 0x8512e0 in operator delete(void*) /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-ubuntu-14-04/toolchain/source/llvm/llvm-3.8.0.src-p1/projects/compiler-rt/lib/asan/asan_new_delete.cc:94
#1 0x7ff0ab0951e2 in boost::scoped_ptr<impala::MemTracker>::reset(impala::MemTracker*) /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/smart_ptr/scoped_ptr.hpp:88:9
#2 0x7ff0ab2bb9a5 in impala::RuntimeState::ReleaseResources() /home/lv/i3/be/src/runtime/runtime-state.cc:285:3
#3 0x7ff0ab246a54 in impala::FragmentInstanceState::Close() /home/lv/i3/be/src/runtime/fragment-instance-state.cc:308:3
#4 0x7ff0ab2434aa in impala::FragmentInstanceState::Exec() /home/lv/i3/be/src/runtime/fragment-instance-state.cc:95:3
#5 0x7ff0ab27cf04 in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) /home/lv/i3/be/src/runtime/query-state.cc:330:19
#6 0x7ff0ab8aa6d2 in boost::function0<void>::operator()() const /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/function/function_template.hpp:766:14
#7 0x7ff0ab8a718d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /home/lv/i3/be/src/util/thread.cc:322:3
#8 0x7ff0ab8b1d8a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind.hpp:457:9
#9 0x7ff0ab8b1c17 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind_template.hpp:20:16
#10 0x94ff09 in thread_proxy (/home/lv/i3/be/build/debug/service/impalad+0x94ff09)
previously allocated by thread T98430 here:
#0 0x850ce0 in operator new(unsigned long) /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-ubuntu-14-04/toolchain/source/llvm/llvm-3.8.0.src-p1/projects/compiler-rt/lib/asan/asan_new_delete.cc:62
#1 0x7ff0ab2b8f78 in impala::RuntimeState::Init() /home/lv/i3/be/src/runtime/runtime-state.cc:124:31
#2 0x7ff0ab2b8912 in impala::RuntimeState::RuntimeState(impala::QueryState*, impala::TPlanFragmentCtx const&, impala::TPlanFragmentInstanceCtx const&, impala::ExecEnv*) /home/lv/i3/be/src/runtime/runtime-state.cc:86:3
#3 0x7ff0ab243a82 in impala::FragmentInstanceState::Prepare() /home/lv/i3/be/src/runtime/fragment-instance-state.cc:119:40
#4 0x7ff0ab24326c in impala::FragmentInstanceState::Exec() /home/lv/i3/be/src/runtime/fragment-instance-state.cc:73:19
#5 0x7ff0ab27cf04 in impala::QueryState::ExecFInstance(impala::FragmentInstanceState*) /home/lv/i3/be/src/runtime/query-state.cc:330:19
#6 0x7ff0ab8aa6d2 in boost::function0<void>::operator()() const /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/function/function_template.hpp:766:14
#7 0x7ff0ab8a718d in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) /home/lv/i3/be/src/util/thread.cc:322:3
#8 0x7ff0ab8b1d8a in void boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind.hpp:457:9
#9 0x7ff0ab8b1c17 in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() /opt/Impala-Toolchain/boost-1.57.0-p2/include/boost/bind/bind_template.hpp:20:16
#10 0x94ff09 in thread_proxy (/home/lv/i3/be/build/debug/service/impalad+0x94ff09)