Uploaded image for project: 'IMPALA'
  1. IMPALA
  2. IMPALA-3385

RuntimeState::error_log_ crashes

    XMLWordPrintableJSON

Details

    Description

      stack trace:

      #0  0x00000032f0032625 in raise () from ss_011/lib64/libc.so.6
      #1  0x00000032f0033e05 in abort () from ss_011/lib64/libc.so.6
      #2  0x00007f52a7eb7a55 in os::abort(bool) () from ss_011/usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #3  0x00007f52a8037f87 in VMError::report_and_die() () from ss_011/usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #4  0x00007f52a7ebc96f in JVM_handle_linux_signal () from ss_011/usr/java/jdk1.7.0_67-cloudera/jre/lib/amd64/server/libjvm.so
      #5  <signal handler called>
      #6  0x00000032f0089a41 in memcpy () from ss_011/lib64/libc.so.6
      #7  0x00007f52a742951b in std::basic_streambuf<char, std::char_traits<char> >::xsputn(char const*, long) () from ss_011/opt/cloudera/parcels/CDH-5.7.0-1.cdh5.7.0.p0.45/lib/impala/lib/libstdc++.so.6
      #8  0x00007f52a7420a86 in std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long) ()
         from ss_011/opt/cloudera/parcels/CDH-5.7.0-1.cdh5.7.0.p0.45/lib/impala/lib/libstdc++.so.6
      #9  0x0000000000a9c743 in std::operator<< <char, std::char_traits<char>, std::allocator<char> > (__str=..., __os=...) at /usr/src/debug/impala-2.5.0-cdh5.7.0/toolchain/build/gcc-4.9.2/include/c++/4.9.2/bits/basic_string.h:2777
      #10 impala::PrintErrorMap (stream=stream@entry=0x7f40f6182300, errors=std::map with 1 elements = {...}) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/util/error-util.cc:143
      #11 0x0000000000a9d13e in impala::PrintErrorMapToString (errors=std::map with 1 elements = {...}) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/util/error-util.cc:155
      #12 0x000000000095be5e in impala::RuntimeState::ErrorLog (this=0x7f4925928c00) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/runtime/runtime-state.cc:201
      #13 0x0000000000b4f1bf in impala::HdfsScanNode::ProcessSplit (this=this@entry=0x7f50d0a4c000, filter_ctxs=std::vector of length 0, capacity 0, scan_range=scan_range@entry=0x7f4e1c983a20) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/exec/hdfs-scan-node.cc:1197
      #14 0x0000000000b5025e in impala::HdfsScanNode::ScannerThread (this=0x7f50d0a4c000) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/exec/hdfs-scan-node.cc:1063
      #15 0x0000000000af8817 in boost::function0<void>::operator() (this=0x7f40f6182d40) at /usr/src/debug/impala-2.5.0-cdh5.7.0/toolchain/build/boost-1.57.0/include/boost/function/function_template.hpp:767
      #16 impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*) (name=..., category=..., functor=..., thread_started=0x7f410b6669e0) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/util/thread.cc:316
      #17 0x0000000000af9134 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0> (
          f=@0x7f505cc4ddb8: 0xaf85f0 <impala::Thread::SuperviseThread(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*)>, a=<synthetic pointer>, this=0x7f505cc4ddc0)
          at /usr/src/debug/impala-2.5.0-cdh5.7.0/toolchain/build/boost-1.57.0/include/boost/bind/bind.hpp:457
      #18 boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > >::operator()() (this=0x7f505cc4ddb8) at /usr/src/debug/impala-2.5.0-cdh5.7.0/toolchain/build/boost-1.57.0/include/boost/bind/bind_template.hpp:20
      #19 boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void ()>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void ()> >, boost::_bi::value<impala::Promise<long>*> > > >::run() (this=0x7f505cc4dc00) at /usr/src/debug/impala-2.5.0-cdh5.7.0/toolchain/build/boost-1.57.0/include/boost/thread/detail/thread.hpp:116
      #20 0x0000000000d3298a in ?? ()
      #21 0x00000032f0407a51 in start_thread () from ss_011/lib64/libpthread.so.0
      #22 0x00000032f00e893d in clone () from ss_011/lib64/libc.so.6
      

      and we have

      (gdb) f 12
      #12 0x000000000095be5e in impala::RuntimeState::ErrorLog (this=0x7f4925928c00) at /usr/src/debug/impala-2.5.0-cdh5.7.0/be/src/runtime/runtime-state.cc:201
      201	  return PrintErrorMapToString(error_log_);
      (gdb) p error_log_
      $3 = std::map with 1 elements = {
        [impala::TErrorCode::HDFS_SCAN_NODE_UNKNOWN_DISK] = {
          _vptr.TErrorLogEntry = 0x207bd90 <vtable for impala::TErrorLogEntry+16>, 
          static ascii_fingerprint = 0x16fded0 "B8D94EBFFE99C6E7010418EB2DF36DC3", 
          static binary_fingerprint =       "\270\331N\277\376\231\306\347\001\004\030\353-\363", <incomplete sequence \303>, 
          count = 0, 
          messages = std::vector of length 0, capacity 1, 
          __isset = {
            count = false, 
            messages = false
          }
        }
      }
      

      so the count is zero, and vector of message is empty.

      In printerrormaptostring we have

      BOOST_FOREACH(const ErrorLogMap::value_type& v, errors) {
          if (v.first == TErrorCode::GENERAL) {
            BOOST_FOREACH(const string& s, v.second.messages) {
              *stream << s << "\n";
            }
          } else {
            *stream << v.second.messages.front();   <----------
            if (v.second.count < 2) {
              *stream << "\n";
            } else {
              *stream << " (1 of " << v.second.count << " similar)\n";
            }
          }
        }
      

      Referring to vecter front is undefined if the vector is empty, thus may crash impala.

      In http://github.mtv.cloudera.com/CDH/Impala/blob/cdh5-2.5.0_5.7.0/be/src/exec/hdfs-scan-node.cc#L1184-L1203

      we check !runtime_state_->error_log().empty(), but empty error log may contain cleared messages, see https://github.com/cloudera/Impala/blob/cdh5-2.5.0_5.7.0/be/src/runtime/runtime-state.cc#L233-L242.

      Later impala crashes on runtime_state_->ErrorLog().

      Attachments

        Activity

          People

            HuaisiXu Huaisi Xu
            HuaisiXu Huaisi Xu
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: