Details
-
Bug
-
Status: Resolved
-
Blocker
-
Resolution: Works for Me
-
Impala 1.2.3
-
None
Description
I've not seen anything incriminating in the logs.
Core files (and the debug binary) can he found here: p0434.mtv.cloudera.com:/data/1/impala-cores/impala-master-cdh4-feb-11
Core #1
#0 0x0000003f51a328e5 in raise () from /lib64/libc.so.6 #1 0x0000003f51a340c5 in abort () from /lib64/libc.so.6 #2 0x0000003f552bea5d in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib64/libstdc++.so.6 #3 0x0000003f552bcbe6 in ?? () from /usr/lib64/libstdc++.so.6 #4 0x0000003f552bcc13 in std::terminate() () from /usr/lib64/libstdc++.so.6 #5 0x0000003f552bcd0e in __cxa_throw () from /usr/lib64/libstdc++.so.6 #6 0x0000000000d0e7e8 in boost::throw_exception<boost::lock_error> (e=...) at /usr/include/boost/throw_exception.hpp:61 #7 0x0000000000d0dbc5 in boost::mutex::lock (this=0x29f9880) at /usr/include/boost/thread/pthread/mutex.hpp:55 #8 0x0000000000d0e944 in boost::lock_guard<boost::mutex>::lock_guard (this=0x7f0863697ca0, m_=...) at /usr/include/boost/thread/locks.hpp:257 #9 0x0000000000e09bcf in impala::MemTracker::~MemTracker (this=0x4f55d40, __in_chrg=<value optimized out>) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/runtime/mem-tracker.cc:132 #10 0x0000000000ddf262 in boost::checked_delete<impala::MemTracker> (x=0x4f55d40) at /usr/include/boost/checked_delete.hpp:34 #11 0x0000000000ddb6cf in boost::scoped_ptr<impala::MemTracker>::~scoped_ptr (this=0x7f0863697d30, __in_chrg=<value optimized out>) at /usr/include/boost/smart_ptr/scoped_ptr.hpp:80 #12 0x0000000000ddbf5b in boost::scoped_ptr<impala::MemTracker>::reset (this=0x1b994f68, p=0x0) at /usr/include/boost/smart_ptr/scoped_ptr.hpp:86 #13 0x0000000000e22562 in impala::RuntimeState::~RuntimeState (this=0x1b994a00, __in_chrg=<value optimized out>) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/runtime/runtime-state.cc:82 #14 0x0000000000f1adcf in boost::checked_delete<impala::RuntimeState> (x=0x1b994a00) at /usr/include/boost/checked_delete.hpp:34 #15 0x0000000000f06737 in boost::scoped_ptr<impala::RuntimeState>::~scoped_ptr (this=0x21fe6ba0, __in_chrg=<value optimized out>) at /usr/include/boost/smart_ptr/scoped_ptr.hpp:80 #16 0x00000000014f98ab in impala::PlanFragmentExecutor::~PlanFragmentExecutor (this=0x21fe6a38, __in_chrg=<value optimized out>) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/runtime/plan-fragment-executor.cc:69 #17 0x0000000000efe0d8 in impala::ImpalaServer::FragmentExecState::~FragmentExecState (this=0x21fe6a00, __in_chrg=<value optimized out>) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/service/fragment-exec-state.h:46 #18 0x0000000000f3ea52 in boost::checked_delete<impala::ImpalaServer::FragmentExecState> (x=0x21fe6a00) at /usr/include/boost/checked_delete.hpp:34 #19 0x0000000000f81fde in boost::detail::sp_counted_impl_p<impala::ImpalaServer::FragmentExecState>::dispose (this=0x29c923a0) at /usr/include/boost/smart_ptr/detail/sp_counted_impl.hpp:78 #20 0x0000000000d0d54c in boost::detail::sp_counted_base::release (this=0x29c923a0) at /usr/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp:145 #21 0x0000000000d0d655 in boost::detail::shared_count::~shared_count (this=0x172899d8, __in_chrg=<value optimized out>) at /usr/include/boost/smart_ptr/detail/shared_count.hpp:217 #22 0x0000000000effb1a in boost::shared_ptr<impala::ImpalaServer::FragmentExecState>::~shared_ptr (this=0x172899d0, __in_chrg=<value optimized out>) at /usr/include/boost/smart_ptr/shared_ptr.hpp:169 #23 0x0000000000effe3d in std::pair<impala::TUniqueId const, boost::shared_ptr<impala::ImpalaServer::FragmentExecState> >::~pair (this=0x172899b8, __in_chrg=<value optimized out>) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_pair.h:68 #24 0x0000000000f562b6 in boost::unordered_detail::destroy<std::pair<impala::TUniqueId const, boost::shared_ptr<impala::ImpalaServer::FragmentExecState> > > (x=0x172899b8) at /usr/include/boost/unordered/detail/fwd.hpp:86 #25 0x0000000000f3f1b8 in boost::unordered_detail::hash_buckets<std::allocator<std::pair<impala::TUniqueId const, boost::shared_ptr<impala::ImpalaServer::FragmentExecState> > >, boost::unordered_detail::ungrouped>::delete_node (this=0x582d730, b=0x172899b0) at /usr/include/boost/unordered/detail/buckets.hpp:67 #26 0x0000000000f23eec in boost::unordered_detail::hash_table<boost::unordered_detail::map<impala::TUniqueId, boost::hash<impala::TUniqueId>, std::equal_to<impala::TUniqueId>, std::allocator<std::pair<impala::TUniqueId const, boost::shared_ptr<impala::ImpalaServer::FragmentExecState> > > > >::erase_return_iterator (this=0x582d730, r=...) at /usr/include/boost/unordered/detail/table.hpp:709 #27 0x0000000000f0cd10 in boost::unordered_map<impala::TUniqueId, boost::shared_ptr<impala::ImpalaServer::FragmentExecState>, boost::hash<impala::TUniqueId>, std::equal_to<impala::TUniqueId>, std::allocator<std::pair<impala::TUniqueId const, boost::shared_ptr<impala::ImpalaServer::FragmentExecState> > > >::erase (this=0x582d730, position=...) at /usr/include/boost/unordered/unordered_map.hpp:370 #28 0x0000000000ed8c28 in impala::ImpalaServer::RunExecPlanFragment (this=0x582d600, exec_state=0x21fe6a00) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/service/impala-server.cc:1080 #29 0x0000000000f7e108 in boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>::operator() (this=0x2eeddd80, p=0x582d600, a1=0x21fe6a00) at /usr/include/boost/bind/mem_fn_template.hpp:165 #30 0x0000000000f7b12d in boost::_bi::list2<boost::_bi::value<impala::ImpalaServer*>, boost::_bi::value<impala::ImpalaServer::FragmentExecState*> >::operator()<boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>, boost::_bi::list0> ( this=0x2eeddd90, f=..., a=...) at /usr/include/boost/bind/bind.hpp:313 #31 0x0000000000f744e3 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::ImpalaServer*>, boost::_bi::value<impala::ImpalaServer::FragmentExecState*> > >::operator() ( this=0x2eeddd80) at /usr/include/boost/bind/bind_template.hpp:20 #32 0x0000000000f69c43 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::ImpalaServer*>, boost::_bi::value<impala::ImpalaServer::FragmentExecState*> > >, void>::invoke (function_obj_ptr=...) at /usr/include/boost/function/function_template.hpp:153 #33 0x0000000000e0caa1 in boost::function0<void>::operator() (this=0x7f0863698cd0) at /usr/include/boost/function/function_template.hpp:1013 #34 0x0000000001167c60 in impala::Thread::SuperviseThread (name="exec-plan-fragment", category="impala-server", functor=..., thread_started=0x7f084b055c60) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/util/thread.cc:263 #35 0x0000000001176e64 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::string &, const std::string &, impala::Thread::ThreadFunctor, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x2bcef810, f=@0x2bcef808, a=...) at /usr/include/boost/bind/bind.hpp:457 #36 0x0000000001176d1b in boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x2bcef808) at /usr/include/boost/bind/bind_template.hpp:20 #37 0x0000000001176c38 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x2bcef680) at /usr/include/boost/thread/detail/thread.hpp:61
Core #2:
#8 0x0000000000d45f03 in std::_Construct<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > (__p=0x24f532b8, __value=Traceback (most recent call last): File "/usr/lib64/../share/gdb/python/libstdcxx/v6/printers.py", line 558, in to_string return self.val['_M_dataplus']['_M_p'].lazy_string (length = len) RuntimeError: Cannot access memory at address 0xffffffffffffffe6 ) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_construct.h:80 #9 0x0000000000d402f0 in std::__uninitialized_copy<false>::uninitialized_copy<__gnu_cxx::__normal_iterator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, std::vector<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >*> (__first=Traceback (most recent call last): File "/usr/lib64/../share/gdb/python/libstdcxx/v6/printers.py", line 558, in to_string return self.val['_M_dataplus']['_M_p'].lazy_string (length = len) RuntimeError: Cannot access memory at address 0xffffffffffffffe6 , __last="/var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala", __result=0x24f532b8) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_uninitialized.h:74 #10 0x0000000000d36a2f in std::uninitialized_copy<__gnu_cxx::__normal_iterator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, std::vector<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >*> (__first=Traceback (most recent call last): File "/usr/lib64/../share/gdb/python/libstdcxx/v6/printers.py", line 558, in to_string return self.val['_M_dataplus']['_M_p'].lazy_string (length = len) RuntimeError: Cannot access memory at address 0xffffffffffffffe6 , __last="/var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala", __result=0x24f532b8) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_uninitialized.h:116 #11 0x0000000000d2dc45 in std::__uninitialized_copy_a<__gnu_cxx::__normal_iterator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const*, std::vector<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >*, std::basic_string<char, std::char_traits<char>, std::allocator<char> > > (__first=Traceback (most recent call last): File "/usr/lib64/../share/gdb/python/libstdcxx/v6/printers.py", line 558, in to_string return self.val['_M_dataplus']['_M_p'].lazy_string (length = len) RuntimeError: Cannot access memory at address 0xffffffffffffffe6 , __last="/var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala", __result=0x24f532b8) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_uninitialized.h:256 #12 0x0000000000d26961 in std::vector<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >::vector (this=0x1de10c68, __x=std::vector of length 1, capacity 1 = {...}) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_vector.h:243 #13 0x0000000000d208f7 in impala::Status::ErrorDetail::ErrorDetail (this=0x1de10c60) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/common/status.h:160 #14 0x0000000000d83054 in impala::Status::operator= (this=0x6d2f358, status=...) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/common/status.h:87 #15 0x0000000000e7a1df in impala::DiskIoMgr::ReaderContext::Cancel (this=0x6d2f2c0, status=...) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/runtime/disk-io-mgr-reader-context.cc:32 #16 0x0000000000e61f9e in impala::DiskIoMgr::CancelReader (this=0x4e1ca00, reader=0x6d2f2c0) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/runtime/disk-io-mgr.cc:345 #17 0x0000000001568ee1 in impala::HdfsScanNode::SetDone (this=0x8cfe680) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/hdfs-scan-node.cc:762 #18 0x000000000156757b in impala::HdfsScanNode::Close (this=0x8cfe680, state=0x21a4b000) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/hdfs-scan-node.cc:555 #19 0x00000000015365df in impala::ExecNode::Close (this=0x3596c500, state=0x21a4b000) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/exec-node.cc:147 #20 0x00000000016415f9 in impala::BlockingJoinNode::Close (this=0x3596c500, state=0x21a4b000) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/blocking-join-node.cc:77 #21 0x000000000154515c in impala::HashJoinNode::Close (this=0x3596c500, state=0x21a4b000) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/hash-join-node.cc:124 #22 0x00000000015365df in impala::ExecNode::Close (this=0x6599e00, state=0x21a4b000) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/exec-node.cc:147 #23 0x000000000166311e in impala::AggregationNode::Close (this=0x6599e00, state=0x21a4b000) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/exec/aggregation-node.cc:237 #24 0x0000000001503fbd in impala::PlanFragmentExecutor::Close (this=0x2fa62738) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/runtime/plan-fragment-executor.cc:506 #25 0x000000000102a871 in impala::ImpalaServer::FragmentExecState::Exec (this=0x2fa62700) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/service/fragment-exec-state.cc:51 #26 0x0000000000ed8a9e in impala::ImpalaServer::RunExecPlanFragment (this=0x4543080, exec_state=0x2fa62700) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/service/impala-server.cc:1071 #27 0x0000000000f7e108 in boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>::operator() (this=0x3ef97340, p=0x4543080, a1=0x2fa62700) at /usr/include/boost/bind/mem_fn_template.hpp:165 #28 0x0000000000f7b12d in boost::_bi::list2<boost::_bi::value<impala::ImpalaServer*>, boost::_bi::value<impala::ImpalaServer::FragmentExecState*> >::operator()<boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>, boost::_bi::list0> ( this=0x3ef97350, f=..., a=...) at /usr/include/boost/bind/bind.hpp:313 #29 0x0000000000f744e3 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::ImpalaServer*>, boost::_bi::value<impala::ImpalaServer::FragmentExecState*> > >::operator() ( this=0x3ef97340) at /usr/include/boost/bind/bind_template.hpp:20 #30 0x0000000000f69c43 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::ImpalaServer, impala::ImpalaServer::FragmentExecState*>, boost::_bi::list2<boost::_bi::value<impala::ImpalaServer*>, boost::_bi::value<impala::ImpalaServer::FragmentExecState*> > >, void>::invoke (function_obj_ptr=...) at /usr/include/boost/function/function_template.hpp:153 #31 0x0000000000e0caa1 in boost::function0<void>::operator() (this=0x7f26c3d0ecd0) at /usr/include/boost/function/function_template.hpp:1013 #32 0x0000000001167c60 in impala::Thread::SuperviseThread (name="exec-plan-fragment", category="impala-server", functor=..., thread_started=0x7f26f1bd6c60) at /var/lib/jenkins/workspace/impala-master-64bit-code-coverage/repos/Impala/be/src/util/thread.cc:263 #33 0x0000000001176e64 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::string &, const std::string &, impala::Thread::ThreadFunctor, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x693c550, f=@0x693c548, a=...) at /usr/include/boost/bind/bind.hpp:457 #34 0x0000000001176d1b in boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x693c548) at /usr/include/boost/bind/bind_template.hpp:20 #35 0x0000000001176c38 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::string&, const std::string&, impala::Thread::ThreadFunctor, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x693c3c0) at /usr/include/boost/thread/detail/thread.hpp:61}