Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-2258

Crash: std::less<impala::TErrorCode::type>::operator()(impala::TErrorCode::type const&, impala::TErrorCode::type const&)

    XMLWordPrintableJSON

Details

    • Bug
    • Status: Resolved
    • Critical
    • Resolution: Duplicate
    • Impala 2.3.0
    • None
    • None
    • impalad version 2.3.0-cdh5-INTERNAL DEBUG (build 74d24ebe897144eccd777a0f765ddedde37d6f6a)

    Description

      The kerberized stress test crashed with the following (I haven't seen this before in a non-kerberized cluster)

      [...skipped...]
#5  <signal handler called>
#6  0x00000000010a8036 in std::less<impala::TErrorCode::type>::operator()(impala::TErrorCode::type const&, impala::TErrorCode::type const&) const ()
#7  0x0000000001385154 in std::_Rb_tree<impala::TErrorCode::type, std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry>, std::_Select1st<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> >, std::less<impala::TErrorCode::type>, std::allocator<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >::_M_lower_bound(std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > const*, std::_Rb_tree_node<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > const*, impala::TErrorCode::type const&) const ()
#8  0x0000000001384b2a in std::_Rb_tree<impala::TErrorCode::type, std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry>, std::_Select1st<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> >, std::less<impala::TErrorCode::type>, std::allocator<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >::find(impala::TErrorCode::type const&) const ()
#9  0x0000000001384983 in std::map<impala::TErrorCode::type, impala::TErrorLogEntry, std::less<impala::TErrorCode::type>, std::allocator<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > >::find(impala::TErrorCode::type const&) const ()
#10 0x0000000001384583 in impala::ErrorCount(std::map<impala::TErrorCode::type, impala::TErrorLogEntry, std::less<impala::TErrorCode::type>, std::allocator<std::pair<impala::TErrorCode::type const, impala::TErrorLogEntry> > > const&) ()
#11 0x00000000011e4289 in impala::RuntimeState::LogError(impala::ErrorMsg const&) ()
#12 0x00000000012006f2 in impala::BufferedBlockMgr::WriteComplete(impala::BufferedBlockMgr::Block*, impala::Status const&) ()
#13 0x000000000120fc76 in boost::_mfi::mf2<void, impala::BufferedBlockMgr, impala::BufferedBlockMgr::Block*, impala::Status const&>::operator()(impala::BufferedBlockMgr*, impala::BufferedBlockMgr::Block*, impala::Status const&) const ()
#14 0x000000000120f9c9 in void boost::_bi::list3<boost::_bi::value<impala::BufferedBlockMgr*>, boost::_bi::value<impala::BufferedBlockMgr::Block*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, impala::BufferedBlockMgr, impala::BufferedBlockMgr::Block*, impala::Status const&>, boost::_bi::list1<impala::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, impala::BufferedBlockMgr, impala::BufferedBlockMgr::Block*, impala::Status const&>&, boost::_bi::list1<impala::Status const&>&, int) ()
#15 0x000000000120f44a in void boost::_bi::bind_t<boost::_bi::unspecified, boost::_mfi::mf2<void, impala::BufferedBlockMgr, impala::BufferedBlockMgr::Block*, impala::Status const&>, boost::_bi::list3<boost::_bi::value<impala::BufferedBlockMgr*>, boost::_bi::value<impala::BufferedBlockMgr::Block*>, boost::arg<1> > >::operator()<impala::Status>(impala::Status const&) ()
#16 0x000000000120eaf0 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<boost::_bi::unspecified, boost::_mfi::mf2<void, impala::BufferedBlockMgr, impala::BufferedBlockMgr::Block*, impala::Status const&>, boost::_bi::list3<boost::_bi::value<impala::BufferedBlockMgr*>, boost::_bi::value<impala::BufferedBlockMgr::Block*>, boost::arg<1> > >, void, impala::Status const&>::invoke(boost::detail::function::function_buffer&, impala::Status const&) ()
#17 0x0000000001223f9e in boost::function1<void, impala::Status const&>::operator()(impala::Status const&) const ()
#18 0x000000000122db28 in impala::DiskIoMgr::RequestContext::Cancel(impala::Status const&) ()
#19 0x000000000121ccc4 in impala::DiskIoMgr::GetNextRequestRange(impala::DiskIoMgr::DiskQueue*, impala::DiskIoMgr::RequestRange**, impala::DiskIoMgr::RequestContext**) ()
#20 0x000000000121de94 in impala::DiskIoMgr::WorkLoop(impala::DiskIoMgr::DiskQueue*) ()
#21 0x000000000122bc6a in boost::_mfi::mf1<void, impala::DiskIoMgr, impala::DiskIoMgr::DiskQueue*>::operator()(impala::DiskIoMgr*, impala::DiskIoMgr::DiskQueue*) const ()
#22 0x000000000122b847 in void boost::_bi::list2<boost::_bi::value<impala::DiskIoMgr*>, boost::_bi::value<impala::DiskIoMgr::DiskQueue*> >::operator()<boost::_mfi::mf1<void, impala::DiskIoMgr, impala::DiskIoMgr::DiskQueue*>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, impala::DiskIoMgr, impala::DiskIoMgr::DiskQueue*>&, boost::_bi::list0&, int) ()
#23 0x000000000122ae87 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::DiskIoMgr, impala::DiskIoMgr::DiskQueue*>, boost::_bi::list2<boost::_bi::value<impala::DiskIoMgr*>, boost::_bi::value<impala::DiskIoMgr::DiskQueue*> > >::operator()() ()
#24 0x000000000122a48a in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, impala::DiskIoMgr, impala::DiskIoMgr::DiskQueue*>, boost::_bi::list2<boost::_bi::value<impala::DiskIoMgr*>, boost::_bi::value<impala::DiskIoMgr::DiskQueue*> > >, void>::invoke(boost::detail::function::function_buffer&) ()
#25 0x00000000011d0715 in boost::function0<void>::operator()() const ()
#26 0x00000000013e39e2 in impala::Thread::SuperviseThread(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*) ()
#27 0x00000000013ecb6a in void boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> >::operator()<void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list0&, int) ()
#28 0x00000000013ecab1 in boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() ()
#29 0x00000000013eca70 in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::function<void ()()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void ()()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() ()
#30 0x00000000016df8a3 in ?? ()
#31 0x0000003c83e079d1 in start_thread () from /lib64/libpthread.so.0
#32 0x0000003c83ae89dd in clone () from /lib64/libc.so.6

      The server log doesn't say much

      #
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00000000010a8036, pid=31161, tid=139945377367808
#
# JRE version: Java(TM) SE Runtime Environment (7.0_67-b01) (build 1.7.0_67-b01)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (24.65-b04 mixed mode linux-amd64 compressed oops)
# Problematic frame:
# C  [impalad+0xca8036]  std::less<impala::TErrorCode::type>::operator()(impala::TErrorCode::type const&, impala::TErrorCode::type const&) const+0x14
#
# Core dump written. Default location: /var/log/impalad/core or core.31161
#
# An error report file with more information is saved as:
# /var/log/impalad/hs_err_pid31161.log
#
# If you would like to submit a bug report, please visit:
#   http://bugreport.sun.com/bugreport/crash.jsp
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.

      There is a core at impala-stress-kerberized-4.vpc.cloudera.com:/var/log/impalad/core.31161

      Attachments

        Issue Links

          relates to

          Bug - A problem which impairs or prevents the functions of the product. IMPALA-2252 Crash (likely race) tearing down BufferedBlockMgr on query failure

          • Blocker - Blocks development and/or testing work, production could not run
          • Resolved

          Activity

            People

              mjacobs Matthew Jacobs
              caseyc casey
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: