Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-2954

Crash in PlanFragmentExecutor::Cancel() dereferencing runtime_state_

    Details

      Description

      The EC2 stress cluster hit a crash in PlanFragmentExecutor::Cancel() dereferencing runtime_state_. Henry Robinson, seems most likely to your fragment startup changes. Can you take look?

      (gdb) bt
      #0  0x00000036cf032625 in raise () from /lib64/libc.so.6
      #1  0x00000036cf033d8d in abort () from /lib64/libc.so.6
      #2  0x00000036cf02b74e in __assert_fail_base () from /lib64/libc.so.6
      #3  0x00000036cf02b810 in __assert_fail () from /lib64/libc.so.6
      #4  0x000000000178f221 in boost::scoped_ptr<impala::RuntimeState>::operator-> (this=0x253f92390)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:99
      #5  0x000000000178e664 in impala::PlanFragmentExecutor::Cancel (this=0x253f92228) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/src/runtime/plan-fragment-executor.cc:536
      #6  0x00000000014a85a4 in impala::FragmentMgr::FragmentExecState::Cancel (this=0x253f92000) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/src/service/fragment-exec-state.cc:39
      #7  0x00000000014a09aa in impala::FragmentMgr::CancelPlanFragment (this=0x5c37f80, return_val=..., params=...) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/src/service/fragment-mgr.cc:136
      #8  0x0000000001411d5a in impala::ImpalaInternalService::CancelPlanFragment (this=0x4ed48a0, return_val=..., params=...)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/src/service/impala-internal-service.h:42
      #9  0x00000000016ca4f8 in impala::ImpalaInternalServiceProcessor::process_CancelPlanFragment (this=0x5f407e0, seqid=0, iprot=0x257908540, oprot=0x2480f7840, callContext=0x216763480)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:1057
      #10 0x00000000016c9376 in impala::ImpalaInternalServiceProcessor::dispatchCall (this=0x5f407e0, iprot=0x257908540, oprot=0x2480f7840, fname=..., seqid=0, callContext=0x216763480)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/generated-sources/gen-cpp/ImpalaInternalService.cpp:922
      #11 0x0000000001408cc4 in apache::thrift::TDispatchProcessor::process (this=0x5f407e0, in=..., out=..., connectionContext=0x216763480)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/thrift-0.9.0-p2/include/thrift/TDispatchProcessor.h:121
      #12 0x000000000217e68f in apache::thrift::server::TThreadedServer::Task::run() ()
      #13 0x00000000012c6289 in impala::ThriftThread::RunRunnable (this=0x11b259740, runnable=..., promise=0x7f687b8ec570) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/src/rpc/thrift-thread.cc:61
      #14 0x00000000012c79d9 in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (this=0x159086fc0, p=0x11b259740,
          a1=..., a2=0x7f687b8ec570) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:280
      #15 0x00000000012c786f in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0x159086fd0, f=..., a=...)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/bind/bind.hpp:392
      #16 0x00000000012c75bb in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0x159086fc0)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #17 0x00000000012c74ce in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (function_obj_ptr=...) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/function/function_template.hpp:153
      #18 0x00000000012fcd1c in boost::function0<void>::operator() (this=0x7f63a991acc0) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/function/function_template.hpp:767
      #19 0x0000000001561a01 in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f687b8ec360) at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/be/src/util/thread.cc:316
      #20 0x00000000015690a4 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x946ecbc0, f=@0x946ecbb8, a=...)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/bind/bind.hpp:457
      #21 0x0000000001568fe7 in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x946ecbb8)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #22 0x0000000001568faa in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x946eca00)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0-SNAPSHOT/toolchain/build/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
      #23 0x000000000182f27a in thread_proxy ()
      #24 0x00000036cf4079d1 in start_thread () from /lib64/libpthread.so.0
      #25 0x00000036cf0e88fd in clone () from /lib64/libc.so.6
      
      (gdb) f 5
      (gdb) p runtime_state_
      $1 = {px = 0x0}
      

        Attachments

          Activity

            People

            • Assignee:
              henryr Henry Robinson
              Reporter:
              dhecht Dan Hecht
            • Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: