[IMPALA-3536] Stress test crash: DCHECK in RuntimeProfile

• Type: Bug
• Status: Resolved
• Priority: Blocker
• Resolution: Cannot Reproduce
• Affects Version/s: Impala 2.6.0
• Component/s: Backend
• Labels: crash
• Target Version: Impala 2.6.0
• Assignee: Matthew Jacobs
• Reporter: Tim Armstrong
• Created: 13/May/16 01:20
• Updated: 17/Jun/16 00:40
• Resolved: 17/May/16 20:29

Description

We encountered a crash running the stress test on the physical cluster with a debug build. THe DCHECK is:

I'll collect more info.

18:11:59 F0512 18:09:29.362815 25496 runtime-profile.cc:170] Check failed: dst_iter->second->unit() == src_iter->second->unit() 
18:11:59 [...skipped...]
18:11:59 #72 0x00000000027af0fa in tc_free ()
18:11:59 #73 0x00007ffb23eae114 in std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_stringstream() () at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/gcc/build/x86_64-unknown-linux-gnu/libstdc++-v3/include/bits/basic_string.h:249
18:11:59 #74 0x00000000014fa629 in impala::PrintId (id=..., separator=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/src/util/debug-util.cc:101
18:11:59 #75 0x00000000014fa4dd in impala::operator<< (os=..., id=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/src/util/debug-util.cc:94
18:11:59 #76 0x00000000014aa889 in impala::FragmentMgr::ExecPlanFragment (this=0xd66bc80, exec_params=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/src/service/fragment-mgr.cc:37
18:11:59 #77 0x000000000140e881 in impala::ImpalaInternalService::ExecPlanFragment (this=0xdd80600, return_val=..., params=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/src/service/impala-internal-service.h:37
18:11:59 #78 0x000000000180dfa6 in impala::ImpalaInternalServiceProcessor::process_ExecPlanFragment (this=0xd666d80, seqid=0, iprot=0x7db31e40, oprot=0x7db31e80, callContext=0x7b3ff140) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1397
18:11:59 #79 0x000000000180dcf4 in impala::ImpalaInternalServiceProcessor::dispatchCall (this=0xd666d80, iprot=0x7db31e40, oprot=0x7db31e80, fname=..., seqid=0, callContext=0x7b3ff140) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1370
18:11:59 #80 0x00000000010f6cba in apache::thrift::TDispatchProcessor::process (this=0xd666d80, in=..., out=..., connectionContext=0x7b3ff140) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/thrift-0.9.0-p4/include/thrift/TDispatchProcessor.h:121
18:11:59 #81 0x00000000026e504f in apache::thrift::server::TThreadedServer::Task::run() ()
18:11:59 #82 0x00000000012af72d in impala::ThriftThread::RunRunnable (this=0x7db31ec0, runnable=..., promise=0x7ffabf66b510) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/src/rpc/thrift-thread.cc:61
18:11:59 #83 0x00000000012b0e7d in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (this=0x420d5710, p=0x7db31ec0, a1=..., a2=0x7ffabf66b510) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:280
18:11:59 #84 0x00000000012b0d13 in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0x420d5720, f=..., a=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:392
18:11:59 #85 0x00000000012b0a5f in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0x420d5710) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
18:11:59 #86 0x00000000012b0972 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (function_obj_ptr=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:153
18:11:59 #87 0x00000000012cf460 in boost::function0<void>::operator() (this=0x7ff9bda71c60) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:767
18:11:59 #88 0x000000000156bbbf in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7ffabf66b300) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/be/src/util/thread.cc:315
18:11:59 #89 0x0000000001572722 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x81705bc0, f=@0x81705bb8, a=...) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:457
18:11:59 #90 0x0000000001572665 in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x81705bb8) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
18:11:59 #91 0x0000000001572628 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x81705a00) at /usr/src/debug/impala-2.6.0-cdh5.8.0-SNAPSHOT/toolchain/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
18:11:59 #92 0x000000000198b34a in thread_proxy ()
18:11:59 #93 0x000000323fa079d1 in start_thread () from /lib64/libpthread.so.0
18:11:59 #94 0x000000323f6e88fd in clone () from /lib64/libc.so.6

Issue Links

• relates to: IMPALA-3757 Crash during impala::RuntimeProfile::ComputeTimeInProfile()