Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Fixed
-
Impala 2.2
-
None
-
None
Description
There were 10 randomly chosen TPCH queries executing concurrently on the 10 node CDH5 cluster on tpch300gb_parquet DB.
One of the nodes crashed and produced this stack trace:
#0 0x00000036f28328e5 in raise () from /lib64/libc.so.6 #0 0x00000036f28328e5 in raise () from /lib64/libc.so.6 #1 0x00000036f28340c5 in abort () from /lib64/libc.so.6 #2 0x0000000001f0de09 in google::DumpStackTraceAndExit () at src/utilities.cc:147 #3 0x0000000001f0542d in google::LogMessage::Fail () at src/logging.cc:1315 #4 0x0000000001f08fc7 in google::LogMessage::SendToLog (this=0x7f1db0f2c990) at src/logging.cc:1269 #5 0x0000000001f079fa in google::LogMessage::Flush (this=0x7f1db0f2c990) at src/logging.cc:1138 #6 0x0000000001f07ccd in google::LogMessageFatal::~LogMessageFatal (this=0x7f1db0f2c990, __in_chrg=<value optimized out>) at src/logging.cc:1836 #7 0x0000000001f6a7cc in impala::FreePool::CheckValidAllocation (this=0x9a7fd40, computed_list_ptr=0x0, allocation=0x7f2500abb9f0 "") at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/runtime/free-pool.h:150 #8 0x0000000001f6a4f4 in impala::FreePool::Free (this=0x9a7fd40, ptr=0x7f2500abb9f0 "") at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/runtime/free-pool.h:92 #9 0x0000000001f69f1e in impala::FunctionContextImpl::FreeLocalAllocations (this=0xb6b1d40) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/udf/udf.cc:423 #10 0x0000000001f683e6 in impala::FunctionContextImpl::Close (this=0xb6b1d40) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/udf/udf.cc:186 #11 0x0000000001524c48 in impala::PartitionedAggregationNode::Partition::Close (this=0x55010b40, finalize_rows=true) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/partitioned-aggregation-node.cc:564 #12 0x000000000152322a in impala::PartitionedAggregationNode::Close (this=0xd2336300, state=0xc3eac700) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/exec/partitioned-aggregation-node.cc:378 #13 0x000000000143ed75 in impala::PlanFragmentExecutor::Close (this=0xa3c1fd10) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/runtime/plan-fragment-executor.cc:570 #14 0x00000000010966f6 in impala::FragmentMgr::FragmentExecState::Exec (this=0xa3c1fb00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/service/fragment-exec-state.cc:50 #15 0x000000000108db1c in impala::FragmentMgr::FragmentExecThread (this=0x948c600, exec_state=0xa3c1fb00) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/service/fragment-mgr.cc:70 #16 0x00000000010920f0 in boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>::operator() (this=0x40a77b40, p=0x948c600, a1=0xa3c1fb00) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/mem_fn_template.hpp:165 #17 0x0000000001091ea9 in boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> >::operator()<boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list0> (this=0x40a77b50, f=..., a=...) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind.hpp:313 #18 0x0000000001091791 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >::operator() (this=0x40a77b40) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind_template.hpp:20 #19 0x00000000010910df in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::FragmentMgr::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::FragmentMgr::FragmentExecState*> > >, void>::invoke (function_obj_ptr=...) at /opt/toolchain/boost-pic-1.55.0/include/boost/function/function_template.hpp:153 #20 0x0000000000f21725 in boost::function0<void>::operator() (this=0x7f1db0f2dcc0) at /opt/toolchain/boost-pic-1.55.0/include/boost/function/function_template.hpp:767 #21 0x000000000117a090 in impala::Thread::SuperviseThread (name="exec-plan-fragment", category="impala-server", functor=..., thread_started=0x7f25b089ab90) at /usr/src/debug/impala-2.2.0-cdh5.5.0-SNAPSHOT/be/src/util/thread.cc:311 #22 0x000000000118322c in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::string &, const std::string &, impala::Thread::ThreadFunctor, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x23faa9c0, f=@0x23faa9b8, a=...) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind.hpp:457 #23 0x0000000001183173 in boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x23faa9b8) at /opt/toolchain/boost-pic-1.55.0/include/boost/bind/bind_template.hpp:20 #24 0x0000000001183132 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x23faa800) at /opt/toolchain/boost-pic-1.55.0/include/boost/thread/detail/thread.hpp:117 #25 0x00000000015e3793 in thread_proxy () #26 0x00000036f2c07851 in start_thread () from /lib64/libpthread.so.0 #27 0x00000036f28e894d in clone () from /lib64/libc.so.6
Additional information is located on e1119.halxg.cloudera.com in /data/4/IMPALA-1864 directory.
This DCHECK failed again in a different run. More info on e1119.halxg.cloudera.com in /data/4/IMPALA-1864