Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-4517

Stress test crash: impala::LlvmCodeGen::FinalizeModule

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Blocker
    • Resolution: Fixed
    • Affects Version/s: Impala 2.8.0
    • Fix Version/s: Impala 2.8.0
    • Component/s: Backend
    • Labels:

      Description

      The stress was running against a Kudu cluster on EC2.

      Stack Trace:

      #0  0x00000032a5a32625 in raise () from /lib64/libc.so.6
      #1  0x00000032a5a33e05 in abort () from /lib64/libc.so.6
      #2  0x00007f6bb13bba55 in os::abort(bool) ()
         from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #3  0x00007f6bb153bf87 in VMError::report_and_die() ()
         from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #4  0x00007f6bb13c096f in JVM_handle_linux_signal ()
         from /usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #5  <signal handler called>
      #6  0x00000000025aec00 in llvm::Comdat::getName() const ()
      #7  0x00000000027187b1 in (anonymous namespace)::Verifier::verify(llvm::Module const&) ()
      #8  0x0000000002718a4d in (anonymous namespace)::VerifierLegacyPass::doFinalization(llvm::Module&) ()
      #9  0x00000000026d2b3c in llvm::FPPassManager::doFinalization(llvm::Module&) ()
      #10 0x00000000026dd26f in llvm::legacy::PassManagerImpl::run(llvm::Module&) ()
      #11 0x000000000235ed39 in llvm::MCJIT::emitObject(llvm::Module*) ()
      #12 0x000000000235f4db in llvm::MCJIT::generateCodeForModule(llvm::Module*) ()
      #13 0x000000000235bb90 in llvm::MCJIT::finalizeObject() ()
      #14 0x000000000162415b in impala::LlvmCodeGen::FinalizeModule (this=0x144796d80)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/codegen/llvm-codegen.cc:949
      #15 0x00000000019d4f2e in impala::PlanFragmentExecutor::OptimizeLlvmModule (
          this=0xf733aa90)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/plan-fragment-executor.cc:264
      #16 0x00000000019d5a30 in impala::PlanFragmentExecutor::OpenInternal (this=0xf733aa90)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/plan-fragment-executor.cc:317
      #17 0x00000000019d56ad in impala::PlanFragmentExecutor::Open (this=0xf733aa90)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/plan-fragment-executor.cc:294
      #18 0x000000000152dc7c in impala::FragmentMgr::FragmentExecState::Exec (this=0xf733a700)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/fragment-exec-state.cc:58
      #19 0x000000000152541a in impala::FragmentMgr::FragmentThread (this=0x8e59300, 
          fragment_instance_id=...)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/service/fragment-mgr.cc:86
      #20 0x000000000152919c in boost::_mfi::mf1<void, impala::FragmentMgr, impala::TUniqueId>::operator() (this=0x4091ab40, p=0x8e59300, a1=...)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/mem_fn_template.hpp:165
      #21 0x0000000001528f59 in boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::TUniqueId> >::operator()<boost::_mfi::mf1<void, impala::FragmentMgr, impala::TUniqueId>, boost::_bi::list0> (this=0x4091ab50, f=..., a=...)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:313
      #22 0x0000000001528883 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::TUniqueId>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::TUniqueId> > >::operator() (this=0x4091ab40)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #23 0x0000000001528216 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::FragmentMgr, impala::TUniqueId>, boost::_bi::list2<boost::_bi::value<impala::FragmentMgr*>, boost::_bi::value<impala::TUniqueId> > >, void>::invoke (function_obj_ptr=...)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:153
      #24 0x000000000133ce98 in boost::function0<void>::operator() (this=0x7f68bb6a8c40)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/function/function_template.hpp:767
      #25 0x00000000015e8921 in impala::Thread::SuperviseThread (name=..., category=..., 
          functor=..., thread_started=0x7f6b055698e0)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:317
      #26 0x00000000015ef8fa in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x2aff29c0, f=@0x2aff29b8, a=...)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind.hpp:457
      #27 0x00000000015ef83d in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x2aff29b8)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #28 0x00000000015ef798 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x2aff2800)
          at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
      #29 0x0000000001a3c34a in thread_proxy ()
      #30 0x00000032a5e079d1 in start_thread () from /lib64/libpthread.so.0
      #31 0x00000032a5ae88fd in clone () from /lib64/libc.so.6
      

      The core file can be found here:

      kudu-stress-5.vpc.cloudera.com:/data1/impalad/core.32601
      

        Attachments

        1. hs_err_pid32601.log
          294 kB
          Taras Bobrovytsky

          Activity

            People

            • Assignee:
              kwho Michael Ho
              Reporter:
              tarasbob Taras Bobrovytsky
            • Votes:
              0 Vote for this issue
              Watchers:
              6 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: