Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-4180

Crash: impala::DiskIoRequestContext::Cancel

    Details

      Description

      A crash occurs if the following query is run in a loop:

      for i in {1..1000}; do impala-shell.sh -q "set num_nodes=1; set DISABLE_CODEGEN=1; set NUM_SCANNER_THREADS=1; set RUNTIME_FILTER_MODE=0; with t as (select int_col x, bigint_col y from functional.alltypestiny order by id limit 2) select * from t t1 left outer join t t2 on t1.y = t2.x full outer join t t3 on t2.y = t3.x order by t1.x limit 10"; done
      

      The query is run in single-node optimization mode (1 node, 1 scanner thread, no codegen, no runtime filters). The crash is non deterministic, so it can take a few minutes for it to occur.

      Stack Trace:

      #0  0x00007f792b3edc37 in __GI_raise (sig=sig@entry=6)
          at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
      #1  0x00007f792b3f1028 in __GI_abort () at abort.c:89
      #2  0x00007f792d613c55 in os::abort(bool) ()
         from /usr/lib/jvm/java-7-oracle-amd64/jre/lib/amd64/server/libjvm.so
      #3  0x00007f792d795cd7 in VMError::report_and_die() ()
         from /usr/lib/jvm/java-7-oracle-amd64/jre/lib/amd64/server/libjvm.so
      #4  0x00007f792d618b6f in JVM_handle_linux_signal ()
         from /usr/lib/jvm/java-7-oracle-amd64/jre/lib/amd64/server/libjvm.so
      #5  <signal handler called>
      #6  __GI___pthread_mutex_lock (mutex=0x0) at ../nptl/pthread_mutex_lock.c:66
      #7  0x00000000010fb23d in pthread_mutex_lock (m=0x71)
          at /tmp/toolchain/boost-1.57.0/include/boost/thread/pthread/mutex.hpp:62
      #8  boost::mutex::lock (this=0x71)
          at /tmp/toolchain/boost-1.57.0/include/boost/thread/pthread/mutex.hpp:116
      #9  0x0000000001104b4c in boost::lock_guard<boost::mutex>::lock_guard (
          this=0x7f78e7531b50, m_=...)
          at /tmp/toolchain/boost-1.57.0/include/boost/thread/lock_guard.hpp:38
      #10 0x000000000133a913 in impala::DiskIoRequestContext::Cancel (this=0x1, status=...)
          at /home/tbobrovytsky/Impala/be/src/runtime/disk-io-mgr-reader-context.cc:30
      #11 0x0000000001328100 in impala::DiskIoMgr::CancelContext (this=0x9046a00, context=0x1, 
          wait_for_disks_completion=true)
          at /home/tbobrovytsky/Impala/be/src/runtime/disk-io-mgr.cc:457
      #12 0x0000000001327d0e in impala::DiskIoMgr::UnregisterContext (this=0x9046a00, 
          reader=0x1) at /home/tbobrovytsky/Impala/be/src/runtime/disk-io-mgr.cc:425
      #13 0x000000000191bcfd in impala::PlanFragmentExecutor::Close (this=0xb231180)
          at /home/tbobrovytsky/Impala/be/src/runtime/plan-fragment-executor.cc:512
      #14 0x000000000191618d in impala::PlanFragmentExecutor::~PlanFragmentExecutor (
          this=0xb231180, __in_chrg=<optimized out>)
          at /home/tbobrovytsky/Impala/be/src/runtime/plan-fragment-executor.cc:73
      #15 0x00000000018f4027 in boost::checked_delete<impala::PlanFragmentExecutor> (
          x=0xb231180) at /tmp/toolchain/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #16 0x00000000018efd81 in boost::scoped_ptr<impala::PlanFragmentExecutor>::~scoped_ptr (
          this=0x95eac28, __in_chrg=<optimized out>)
          at /tmp/toolchain/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:82
      #17 0x00000000018db887 in impala::Coordinator::~Coordinator (this=0x95ea800, 
          __in_chrg=<optimized out>)
          at /home/tbobrovytsky/Impala/be/src/runtime/coordinator.cc:365
      #18 0x000000000147f5ff in boost::checked_delete<impala::Coordinator> (x=0x95ea800)
          at /tmp/toolchain/boost-1.57.0/include/boost/core/checked_delete.hpp:34
      #19 0x000000000147b121 in boost::scoped_ptr<impala::Coordinator>::~scoped_ptr (
          this=0xaf3e3c0, __in_chrg=<optimized out>)
          at /tmp/toolchain/boost-1.57.0/include/boost/smart_ptr/scoped_ptr.hpp:82
      #20 0x0000000001469330 in impala::ImpalaServer::QueryExecState::~QueryExecState (
          this=0xaf3e000, __in_chrg=<optimized out>)
          at /home/tbobrovytsky/Impala/be/src/service/query-exec-state.cc:115
      #21 0x0000000001433ed6 in std::_Sp_counted_ptr<impala::ImpalaServer::QueryExecState*, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0xb0ac4e0)
          at /tmp/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/shared_ptr_base.h:373
      #22 0x00000000010efb1a in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (
          this=0xb0ac4e0)
          at /tmp/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/shared_ptr_base.h:149
      #23 0x00000000010ee559 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
          (this=0x7f78e75321f8, __in_chrg=<optimized out>)
          at /tmp/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/shared_ptr_base.h:666
      #24 0x0000000001402b94 in std::__shared_ptr<impala::ImpalaServer::QueryExecState, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x7f78e75321f0, __in_chrg=<optimized out>)
          at /tmp/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/shared_ptr_base.h:914
      #25 0x0000000001402bae in std::shared_ptr<impala::ImpalaServer::QueryExecState>::~shared_ptr (this=0x7f78e75321f0, __in_chrg=<optimized out>)
          at /tmp/toolchain/gcc-4.9.2/include/c++/4.9.2/bits/shared_ptr.h:93
      #26 0x00000000013f0927 in impala::ImpalaServer::UnregisterQuery (this=0xa633e00, 
          query_id=..., check_inflight=true, cause=0x0)
          at /home/tbobrovytsky/Impala/be/src/service/impala-server.cc:977
      #27 0x0000000001460640 in impala::ImpalaServer::close (this=0xa633e00, handle=...)
          at /home/tbobrovytsky/Impala/be/src/service/impala-beeswax-server.cc:355
      #28 0x000000000188e5d2 in beeswax::BeeswaxServiceProcessor::process_close (
          this=0xaec10e0, seqid=0, iprot=0x9da0420, oprot=0x9da18c0, callContext=0xb0e3a00)
          at /home/tbobrovytsky/Impala/be/generated-sources/gen-cpp/BeeswaxService.cpp:3543
      #29 0x00000000018896a2 in beeswax::BeeswaxServiceProcessor::dispatchCall (this=0xaec10e0, 
          iprot=0x9da0420, oprot=0x9da18c0, fname=..., seqid=0, callContext=0xb0e3a00)
          at /home/tbobrovytsky/Impala/be/generated-sources/gen-cpp/BeeswaxService.cpp:2952
      #30 0x000000000187349f in impala::ImpalaServiceProcessor::dispatchCall (this=0xaec10e0, 
          iprot=0x9da0420, oprot=0x9da18c0, fname=..., seqid=0, callContext=0xb0e3a00)
          at /home/tbobrovytsky/Impala/be/generated-sources/gen-cpp/ImpalaService.cpp:1673
      #31 0x00000000010ecdf4 in apache::thrift::TDispatchProcessor::process (this=0xaec10e0, 
          in=..., out=..., connectionContext=0xb0e3a00)
          at /tmp/toolchain/thrift-0.9.0-p8/include/thrift/TDispatchProcessor.h:121
      #32 0x000000000270127b in apache::thrift::server::TThreadPoolServer::Task::run() ()
      #33 0x00000000026e8e59 in apache::thrift::concurrency::ThreadManager::Worker::run() ()
      #34 0x00000000012a7f5f in impala::ThriftThread::RunRunnable (this=0xb0d6bc0, 
          runnable=..., promise=0x7ffc1d58c580)
          at /home/tbobrovytsky/Impala/be/src/rpc/thrift-thread.cc:64
      #35 0x00000000012a96af in boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() (
          this=0xb0e0660, p=0xb0d6bc0, a1=..., a2=0x7ffc1d58c580)
          at /tmp/toolchain/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:280
      #36 0x00000000012a9545 in boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> >::operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list0> (this=0xb0e0670, f=..., a=...)
          at /tmp/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:392
      #37 0x00000000012a9291 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >::operator() (this=0xb0e0660)
          at /tmp/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #38 0x00000000012a91a4 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::val---Type <return> to continue, or q <return> to quit---
      ue<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable> >, boost::_bi::value<impala::Promise<unsigned long>*> > >, void>::invoke (
          function_obj_ptr=...)
          at /tmp/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:153
      #39 0x00000000012ae736 in boost::function0<void>::operator() (this=0x7f78e7532d60)
          at /tmp/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:767
      #40 0x000000000155da85 in impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) (name=..., category=..., 
          functor=..., thread_started=0x7ffc1d58c370)
          at /home/tbobrovytsky/Impala/be/src/util/thread.cc:317
      #41 0x00000000015640dc in boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list0&, int) (this=0xb0ecdc0, 
          f=@0xb0ecdb8: 0x155d7c0 <impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*)>, a=...)
          at /tmp/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:457
      #42 0x000000000156401f in boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() (this=0xb0ecdb8)
          at /tmp/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #43 0x0000000001563f7a in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() (
          this=0xb0ecc00)
          at /tmp/toolchain/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
      #44 0x00000000019801aa in thread_proxy ()
      #45 0x00007f792b784184 in start_thread (arg=0x7f78e7533700) at pthread_create.c:312
      #46 0x00007f792b4b137d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
      

        Attachments

          Activity

            People

            • Assignee:
              kwho Michael Ho
              Reporter:
              tarasbob Taras Bobrovytsky
            • Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: